A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
support bitsandbytes quantization with more models (#9148)
chenqianfzh committed
Oct 8, 2024 at 18:52 UTC
2f4117c38e101ee63b65521c93b22efe3526f77e
10 files changed
+164
-27
tests/quantization/test_bitsandbytes.py
+7
-6
@@ -9,22 +9,22 @@ import pytest
9
import torch
10
11
from tests.quantization.utils import is_quant_method_supported
12
-
13
-from ..utils import fork_new_process_for_each_test
12
+from tests.utils import fork_new_process_for_each_test
13
14
models_4bit_to_test = [
16
- ('huggyllama/llama-7b', 'quantize model inflight'),
15
+ ("facebook/opt-125m", "quantize opt model inflight"),
16
]
17
18
models_pre_qaunt_4bit_to_test = [
20
- ('lllyasviel/omost-llama-3-8b-4bits',
21
- 'read pre-quantized 4-bit NF4 model'),
19
('PrunaAI/Einstein-v6.1-Llama3-8B-bnb-4bit-smashed',
20
'read pre-quantized 4-bit FP4 model'),
21
+ ('poedator/opt-125m-bnb-4bit', 'read pre-quantized 4-bit NF4 opt model'),
22
]
23
24
models_pre_quant_8bit_to_test = [
27
- ('meta-llama/Llama-Guard-3-8B-INT8', 'read pre-quantized 8-bit model'),
25
+ ('meta-llama/Llama-Guard-3-8B-INT8',
26
+ 'read pre-quantized llama 8-bit model'),
27
+ ("yec019/fbopt-350m-8bit", "read pre-quantized 8-bit opt model"),
28
]
29
30
@@ -133,6 +133,7 @@ def validate_generated_texts(hf_runner,
133
hf_str = hf_log["generated_text"]
134
vllm_str = vllm_log["generated_text"]
135
prompt = hf_log["prompt"]
136
+
137
assert hf_str == vllm_str, (f"Model: {model_name}"
138
f"Mismatch between HF and vLLM outputs:\n"
139
f"Prompt: {prompt}\n"
vllm/model_executor/layers/linear.py
+25
-1
@@ -336,8 +336,12 @@ class ColumnParallelLinear(LinearBase):
336
if is_gguf_weight and isinstance(param, UninitializedParameter):
337
param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
338
339
+ use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
340
+
341
param_data = param.data
340
- if output_dim is not None:
342
+ # bitsandbytes loads the weights of the specific portion
343
+ # no need to narrow here
344
+ if output_dim is not None and not use_bitsandbytes_4bit:
345
shard_size = param_data.shape[output_dim]
346
start_idx = tp_rank * shard_size
347
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
@@ -821,6 +825,9 @@ class QKVParallelLinear(ColumnParallelLinear):
825
("v", (self.total_num_heads + self.total_num_kv_heads) *
826
self.head_size, self.total_num_kv_heads * self.head_size),
827
]
828
+ use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
829
+ False)
830
+
831
packed_dim = getattr(param, "packed_dim", None)
832
for shard_id, shard_offset, shard_size in shard_offsets:
833
# Special case for Quantized Weights.
@@ -834,6 +841,23 @@ class QKVParallelLinear(ColumnParallelLinear):
841
shard_size, shard_offset = adjust_marlin_shard(
842
param, shard_size, shard_offset)
843
844
+ if use_bitsandbytes_4bit:
845
+ orig_qkv_offsets = {
846
+ "q": (0, self.total_num_heads * self.head_size),
847
+ "k": (self.total_num_heads * self.head_size,
848
+ self.total_num_kv_heads * self.head_size),
849
+ "v":
850
+ ((self.total_num_heads + self.total_num_kv_heads) *
851
+ self.head_size,
852
+ self.total_num_kv_heads * self.head_size),
853
+ "total":
854
+ ((self.total_num_heads + 2 * self.total_num_kv_heads) *
855
+ self.head_size, 0)
856
+ }
857
+
858
+ shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(
859
+ param, orig_qkv_offsets, shard_id)
860
+
861
loaded_weight_shard = loaded_weight.narrow(
862
output_dim, shard_offset, shard_size)
863
self.weight_loader(param, loaded_weight_shard, shard_id)
vllm/model_executor/layers/quantization/bitsandbytes.py
+2
-2
@@ -108,7 +108,7 @@ class BitsAndBytesConfig(QuantizationConfig):
108
return None
109
110
def get_scaled_act_names(self) -> List[str]:
111
- return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
111
+ return []
112
113
114
class BitsAndBytesLinearMethod(LinearMethodBase):
@@ -236,7 +236,7 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
236
if generation == 0 or generation == 1:
237
matmul_states[i] = MatmulLtState()
238
matmul_states[i].CB = qweight[offsets[i]:offsets[i + 1]]
239
- matmul_states[i].SCB = quant_states[i]
239
+ matmul_states[i].SCB = quant_states[i].to(x.device)
240
matmul_states[i].threshold = (
241
self.quant_config.llm_int8_threshold)
242
matmul_states[i].has_fp16_weights = (
vllm/model_executor/model_loader/loader.py
+44
-18
@@ -736,15 +736,26 @@ class ShardedStateLoader(BaseModelLoader):
736
class BitsAndBytesModelLoader(BaseModelLoader):
737
"""Model loader to load model weights with BitAndBytes quantization."""
738
739
- # TODO: these module names are for Llama only,
740
- # change so that it works with other models as well
739
+ possible_config_file_names = ["adapter_config.json"]
740
+
741
default_target_modules = [
742
- "gate_proj", "down_proj", "up_proj", "q_proj", "k_proj", "v_proj",
743
- "o_proj"
742
+ ".gate_proj.",
743
+ ".down_proj.",
744
+ ".up_proj.",
745
+ ".q_proj.",
746
+ ".k_proj.",
747
+ ".v_proj.",
748
+ ".o_proj.",
749
+ '.fc1.',
750
+ '.fc2.',
751
+ '.dense.',
752
+ '.query_key_value.',
753
+ '.qkv_proj.',
754
+ '.dense_h_to_4h.',
755
+ '.dense_4h_to_h.',
756
+ '.out_proj.',
757
]
758
746
- possible_config_file_names = ["adapter_config.json"]
747
-
759
def __init__(self, load_config: LoadConfig):
760
super().__init__(load_config)
761
@@ -754,7 +765,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
765
if (not load_config.model_loader_extra_config
766
or "qlora_adapter_name_or_path"
767
not in load_config.model_loader_extra_config):
757
- self.target_modules = self.default_target_modules
768
+ self.target_modules = []
769
return
770
771
qlora_adapter = load_config.model_loader_extra_config[
@@ -901,10 +912,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
912
for weight_name, weight_tensor in self._hf_weight_iter(
913
hf_weights_files, use_safetensors):
914
904
- if not weight_name.endswith(".weight"):
915
+ if not weight_name.endswith((".weight", ".bias")):
916
continue
917
918
qweight_name = weight_name.replace(".weight", ".qweight")
919
+
920
if qweight_name in quant_state_dict:
921
set_weight_attrs(weight_tensor, {"load_in_8bit": True})
922
yield qweight_name, weight_tensor
@@ -920,7 +932,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
932
use_safetensors)
933
temp_state_dict = {}
934
for weight_name, weight_tensor in weight_iterator:
923
- if weight_name.endswith(".weight"):
935
+ if weight_name.endswith((".weight", ".bias")):
936
continue
937
# bitsandbytes library requires
938
# weight.quant_state.bitsandbytes__* in CPU
@@ -943,9 +955,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
955
# pre quantized weights would have a quant_state
956
for weight_name, weight_tensor in self._hf_weight_iter(
957
hf_weights_files, use_safetensors):
946
- # Filter out all weights whose suffix is not ".weight"
947
- if not weight_name.endswith(".weight"):
958
+
959
+ if not weight_name.endswith((".weight", ".bias")):
960
continue
961
+
962
if (f"{weight_name}.quant_state.bitsandbytes__nf4" \
963
in temp_state_dict) or \
964
(f"{weight_name}.quant_state.bitsandbytes__fp4" \
@@ -965,15 +978,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
978
979
for weight_name, weight_tensor in self._hf_weight_iter(
980
hf_weights_files, use_safetensors):
968
- if any(target_module in weight_name
969
- for target_module in self.target_modules):
981
+
982
+ if any(target_module in weight_name for target_module in
983
+ self.target_modules) and weight_name.endswith(".weight"):
984
weight_name = weight_name.replace(".weight", ".qweight")
985
972
- # weight partitions of different modules occur at
973
- # different dimensions
974
- # TODO: these module names are for Llama only,
975
- # change so that it works with other models as well
976
- if 'down_proj' in weight_name or 'o_proj' in weight_name:
986
+ if any(module in weight_name
987
+ for module in self.column_parallel_weights_modules):
988
+
989
total_size = weight_tensor.size(-1)
990
start_index = total_size // tp_size * tp_rank
991
end_index = total_size // tp_size * (tp_rank + 1)
@@ -1022,6 +1034,20 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1034
f"Model {type(model).__name__} does not support BitsAndBytes "
1035
"quantization yet.")
1036
1037
+ if len(self.target_modules) == 0:
1038
+ if hasattr(model, 'default_bitsandbytes_target_modules'):
1039
+ self.target_modules = model.default_bitsandbytes_target_modules
1040
+ else:
1041
+ self.target_modules = self.default_target_modules
1042
+
1043
+ if hasattr(model, 'column_parallel_weights_modules'):
1044
+ self.column_parallel_weights_modules = \
1045
+ model.column_parallel_weights_modules
1046
+ else:
1047
+ self.column_parallel_weights_modules = []
1048
+
1049
+ self.model_type = type(model).__name__
1050
+
1051
logger.info("Loading weights with BitsAndBytes quantization. "
1052
" May take a while ...")
1053
vllm/model_executor/models/falcon.py
+11
@@ -391,6 +391,17 @@ class FalconModel(nn.Module):
391
392
class FalconForCausalLM(nn.Module, SupportsPP):
393
394
+ # BitandBytes specific attributes
395
+ bitsandbytes_stacked_params_mapping = {}
396
+ default_bitsandbytes_target_modules = [
397
+ ".query_key_value.",
398
+ ".dense.",
399
+ ".dense_h_to_4h.",
400
+ ".dense_4h_to_h.",
401
+ ]
402
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
403
+ column_parallel_weights_modules = [".dense_4h_to_h.", ".dense."]
404
+
405
def __init__(
406
self,
407
config: FalconConfig,
vllm/model_executor/models/gemma.py
+22
@@ -332,6 +332,28 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332
"gate_up_proj",
333
"down_proj",
334
]
335
+
336
+ # BitandBytes specific attributes
337
+ default_bitsandbytes_target_modules = [
338
+ ".gate_proj.",
339
+ ".down_proj.",
340
+ ".up_proj.",
341
+ ".q_proj.",
342
+ ".k_proj.",
343
+ ".v_proj.",
344
+ ".o_proj.",
345
+ ]
346
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
347
+ column_parallel_weights_modules = [".down_proj.", ".o_proj."]
348
+ bitsandbytes_stacked_params_mapping = {
349
+ # shard_name, weight_name, index
350
+ "q_proj": ("qkv_proj", 0),
351
+ "k_proj": ("qkv_proj", 1),
352
+ "v_proj": ("qkv_proj", 2),
353
+ "gate_proj": ("gate_up_proj", 0),
354
+ "up_proj": ("gate_up_proj", 1),
355
+ }
356
+
357
# Gemma does not apply LoRA to the embedding layer.
358
embedding_modules = {}
359
embedding_padding_modules = []
vllm/model_executor/models/gemma2.py
+13
@@ -375,6 +375,19 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
375
# Gemma does not apply LoRA to the embedding layer.
376
embedding_modules = {}
377
embedding_padding_modules = []
378
+
379
+ # BitandBytes specific attributes
380
+ default_bitsandbytes_target_modules = [
381
+ ".gate_proj.",
382
+ ".down_proj.",
383
+ ".up_proj.",
384
+ ".q_proj.",
385
+ ".k_proj.",
386
+ ".v_proj.",
387
+ ".o_proj.",
388
+ ]
389
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
390
+ column_parallel_weights_modules = [".down_proj.", ".o_proj."]
391
bitsandbytes_stacked_params_mapping = {
392
# shard_name, weight_name, index
393
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/llama.py
+13
@@ -449,6 +449,19 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
449
"lm_head": "output_embeddings"
450
}
451
embedding_padding_modules = ["lm_head"]
452
+
453
+ # BitandBytes specific attributes
454
+ default_bitsandbytes_target_modules = [
455
+ ".gate_proj.",
456
+ ".down_proj.",
457
+ ".up_proj.",
458
+ ".q_proj.",
459
+ ".k_proj.",
460
+ ".v_proj.",
461
+ ".o_proj.",
462
+ ]
463
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
464
+ column_parallel_weights_modules = [".down_proj.", ".o_proj."]
465
bitsandbytes_stacked_params_mapping = {
466
# shard_name, weight_name, index
467
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/opt.py
+13
@@ -315,6 +315,19 @@ class OPTModel(nn.Module):
315
316
class OPTForCausalLM(nn.Module, SupportsPP):
317
318
+ # BitandBytes specific attributes
319
+ bitsandbytes_stacked_params_mapping = {
320
+ # shard_name, weight_name, index
321
+ "q_proj": ("qkv_proj", 0),
322
+ "k_proj": ("qkv_proj", 1),
323
+ "v_proj": ("qkv_proj", 2),
324
+ }
325
+ default_bitsandbytes_target_modules = [
326
+ ".q_proj.", ".k_proj.", ".v_proj.", ".out_proj.", ".fc1.", ".fc2."
327
+ ]
328
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
329
+ column_parallel_weights_modules = [".out_proj.", ".fc2."]
330
+
331
def __init__(
332
self,
333
config: OPTConfig,
vllm/model_executor/models/phi.py
+14
@@ -260,6 +260,20 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
260
"fc1",
261
"fc2",
262
]
263
+
264
+ # BitandBytes specific attributes
265
+ bitsandbytes_stacked_params_mapping = {
266
+ # shard_name, weight_name, index
267
+ "q_proj": ("qkv_proj", 0),
268
+ "k_proj": ("qkv_proj", 1),
269
+ "v_proj": ("qkv_proj", 2),
270
+ }
271
+ default_bitsandbytes_target_modules = [
272
+ ".q_proj.", ".k_proj.", ".v_proj.", ".fc1.", ".fc2.", ".dense."
273
+ ]
274
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
275
+ column_parallel_weights_modules = [".fc2.", ".dense."]
276
+
277
embedding_modules = {}
278
embedding_padding_modules = []
279