A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
support bitsandbytes 8-bit and FP4 quantized models (#7445)
chenqianfzh committed
Aug 29, 2024 at 16:09 UTC
4664ceaad6f99ec7824859d1ac31b29502565a98
6 files changed
+437
-191
tests/conftest.py
+6
@@ -209,8 +209,14 @@ class HfRunner:
209
210
def wrap_device(self, input: _T) -> _T:
211
if not is_cpu():
212
+ # Check if the input is already on the GPU
213
+ if hasattr(input, 'device') and input.device.type == "cuda":
214
+ return input # Already on GPU, no need to move
215
return input.to("cuda")
216
else:
217
+ # Check if the input is already on the CPU
218
+ if hasattr(input, 'device') and input.device.type == "cpu":
219
+ return input # Already on CPU, no need to move
220
return input.to("cpu")
221
222
def __init__(
tests/quantization/test_bitsandbytes.py
+98
-68
@@ -2,85 +2,115 @@
2
3
Run `pytest tests/quantization/test_bitsandbytes.py`.
4
'''
5
+
6
+import gc
7
+
8
import pytest
9
import torch
10
11
from tests.quantization.utils import is_quant_method_supported
9
-from vllm import SamplingParams
12
11
-models_to_test = [
13
+models_4bit_to_test = [
14
('huggyllama/llama-7b', 'quantize model inflight'),
13
- ('lllyasviel/omost-llama-3-8b-4bits', 'read pre-quantized model'),
15
]
16
17
+models_pre_qaunt_4bit_to_test = [
18
+ ('lllyasviel/omost-llama-3-8b-4bits',
19
+ 'read pre-quantized 4-bit NF4 model'),
20
+ ('PrunaAI/Einstein-v6.1-Llama3-8B-bnb-4bit-smashed',
21
+ 'read pre-quantized 4-bit FP4 model'),
22
+]
23
+
24
+models_pre_quant_8bit_to_test = [
25
+ ('meta-llama/Llama-Guard-3-8B-INT8', 'read pre-quantized 8-bit model'),
26
+]
27
+
28
+
29
+@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
30
+ reason='bitsandbytes is not supported on this GPU type.')
31
+@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
32
+def test_load_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
33
+ model_name, description) -> None:
34
+
35
+ hf_model_kwargs = {"load_in_4bit": True}
36
+ validate_generated_texts(hf_runner, vllm_runner, example_prompts[:1],
37
+ model_name, hf_model_kwargs)
38
+
39
+
40
+@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
41
+ reason='bitsandbytes is not supported on this GPU type.')
42
+@pytest.mark.parametrize("model_name, description",
43
+ models_pre_qaunt_4bit_to_test)
44
+def test_load_pre_quant_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
45
+ model_name, description) -> None:
46
+
47
+ validate_generated_texts(hf_runner, vllm_runner, example_prompts[:1],
48
+ model_name)
49
+
50
51
@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
52
reason='bitsandbytes is not supported on this GPU type.')
19
-@pytest.mark.parametrize("model_name, description", models_to_test)
20
-def test_load_bnb_model(vllm_runner, model_name, description) -> None:
53
+@pytest.mark.parametrize("model_name, description",
54
+ models_pre_quant_8bit_to_test)
55
+def test_load_8bit_bnb_model(hf_runner, vllm_runner, example_prompts,
56
+ model_name, description) -> None:
57
+
58
+ validate_generated_texts(hf_runner, vllm_runner, example_prompts[:1],
59
+ model_name)
60
+
61
+
62
+def log_generated_texts(prompts, outputs, runner_name):
63
+ logged_texts = []
64
+ for i, (_, generated_text) in enumerate(outputs):
65
+ log_entry = {
66
+ "prompt": prompts[i],
67
+ "runner_name": runner_name,
68
+ "generated_text": generated_text,
69
+ }
70
+ logged_texts.append(log_entry)
71
+ return logged_texts
72
+
73
+
74
+def validate_generated_texts(hf_runner,
75
+ vllm_runner,
76
+ prompts,
77
+ model_name,
78
+ hf_model_kwargs=None):
79
+
80
+ if hf_model_kwargs is None:
81
+ hf_model_kwargs = {}
82
+
83
+ # Run with HF runner
84
+ with hf_runner(model_name, model_kwargs=hf_model_kwargs) as llm:
85
+ hf_outputs = llm.generate_greedy(prompts, 8)
86
+ hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")
87
+
88
+ # Clean up the GPU memory for the next test
89
+ torch.cuda.synchronize()
90
+ gc.collect()
91
+ torch.cuda.empty_cache()
92
+
93
+ #Run with vLLM runner
94
with vllm_runner(model_name,
95
quantization='bitsandbytes',
96
load_format='bitsandbytes',
24
- enforce_eager=True) as llm:
25
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
26
-
27
- # check the weights in MLP & SelfAttention are quantized to torch.uint8
28
- qweight = model.model.layers[0].mlp.gate_up_proj.qweight
29
- assert qweight.dtype == torch.uint8, (
30
- f'Expected gate_up_proj dtype torch.uint8 but got {qweight.dtype}')
31
-
32
- qweight = model.model.layers[0].mlp.down_proj.qweight
33
- assert qweight.dtype == torch.uint8, (
34
- f'Expected down_proj dtype torch.uint8 but got {qweight.dtype}')
35
-
36
- qweight = model.model.layers[0].self_attn.o_proj.qweight
37
- assert qweight.dtype == torch.uint8, (
38
- f'Expected o_proj dtype torch.uint8 but got {qweight.dtype}')
39
-
40
- qweight = model.model.layers[0].self_attn.qkv_proj.qweight
41
- assert qweight.dtype == torch.uint8, (
42
- f'Expected qkv_proj dtype torch.uint8 but got {qweight.dtype}')
43
-
44
- # some weights should not be quantized
45
- weight = model.lm_head.weight
46
- assert weight.dtype != torch.uint8, (
47
- 'lm_head weight dtype should not be torch.uint8')
48
-
49
- weight = model.model.embed_tokens.weight
50
- assert weight.dtype != torch.uint8, (
51
- 'embed_tokens weight dtype should not be torch.uint8')
52
-
53
- weight = model.model.layers[0].input_layernorm.weight
54
- assert weight.dtype != torch.uint8, (
55
- 'input_layernorm weight dtype should not be torch.uint8')
56
-
57
- weight = model.model.layers[0].post_attention_layernorm.weight
58
- assert weight.dtype != torch.uint8, (
59
- 'input_layernorm weight dtype should not be torch.uint8')
60
-
61
- # check the output of the model is expected
62
- sampling_params = SamplingParams(temperature=0.0,
63
- logprobs=1,
64
- prompt_logprobs=1,
65
- max_tokens=8)
66
-
67
- prompts = ['That which does not kill us', 'To be or not to be,']
68
- expected_outputs = [
69
- 'That which does not kill us makes us stronger.',
70
- 'To be or not to be, that is the question.'
71
- ]
72
- outputs = llm.generate(prompts, sampling_params=sampling_params)
73
- assert len(outputs) == len(prompts)
74
-
75
- for index in range(len(outputs)):
76
- # compare the first line of the output
77
- actual_output = outputs[index][1][0].split('\n', 1)[0]
78
- expected_output = expected_outputs[index].split('\n', 1)[0]
79
-
80
- assert len(actual_output) >= len(expected_output), (
81
- f'Actual {actual_output} should be larger than or equal to '
82
- f'expected {expected_output}')
83
- actual_output = actual_output[:len(expected_output)]
84
-
85
- assert actual_output == expected_output, (
86
- f'Expected: {expected_output}, but got: {actual_output}')
97
+ enforce_eager=True,
98
+ gpu_memory_utilization=0.8) as llm:
99
+ vllm_outputs = llm.generate_greedy(prompts, 8)
100
+ vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")
101
+
102
+ # Clean up the GPU memory for the next test
103
+ torch.cuda.synchronize()
104
+ gc.collect()
105
+ torch.cuda.empty_cache()
106
+
107
+ # Compare the generated strings
108
+ for hf_log, vllm_log in zip(hf_logs, vllm_logs):
109
+ hf_str = hf_log["generated_text"]
110
+ vllm_str = vllm_log["generated_text"]
111
+ prompt = hf_log["prompt"]
112
+ assert hf_str == vllm_str, (f"Model: {model_name}"
113
+ f"Mismatch between HF and vLLM outputs:\n"
114
+ f"Prompt: {prompt}\n"
115
+ f"HF Output: '{hf_str}'\n"
116
+ f"vLLM Output: '{vllm_str}'")
vllm/config.py
+2
@@ -405,6 +405,8 @@ class ModelConfig:
405
raise ValueError(
406
"BitAndBytes quantization with TP or PP is not supported yet.")
407
408
+ # Remove the constraint after the bitsandbytes issue is fixed:
409
+ # https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1308
410
if self.quantization == "bitsandbytes" and self.enforce_eager is False:
411
logger.warning("CUDA graph is not supported on BitAndBytes yet, "
412
"fallback to the eager mode.")
vllm/model_executor/layers/linear.py
+10
-8
@@ -36,9 +36,9 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
36
return shard_size * marlin_tile_size, shard_offset * marlin_tile_size
37
38
39
-def adjust_bitsandbytes_shard(param: Parameter,
40
- qkv_offsets: Dict[str, Tuple[int, int]],
41
- loaded_shard_id: str) -> Tuple[int, int]:
39
+def adjust_bitsandbytes_4bit_shard(param: Parameter,
40
+ qkv_offsets: Dict[str, Tuple[int, int]],
41
+ loaded_shard_id: str) -> Tuple[int, int]:
42
"""Adjust the quantization offsets and sizes for BitsAndBytes sharding."""
43
44
total, _ = qkv_offsets["total"]
@@ -505,8 +505,9 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
505
shard_size, shard_offset = adjust_marlin_shard(
506
param, shard_size, shard_offset)
507
508
- use_bitsandbytes = getattr(param, "use_bitsandbytes", False)
509
- if use_bitsandbytes:
508
+ use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
509
+ False)
510
+ if use_bitsandbytes_4bit:
511
shard_size = loaded_weight.shape[output_dim]
512
shard_offset = loaded_weight.shape[output_dim] * \
513
loaded_shard_id
@@ -858,8 +859,9 @@ class QKVParallelLinear(ColumnParallelLinear):
859
shard_size, shard_offset = adjust_marlin_shard(
860
param, shard_size, shard_offset)
861
861
- use_bitsandbytes = getattr(param, "use_bitsandbytes", False)
862
- if use_bitsandbytes:
862
+ use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
863
+ False)
864
+ if use_bitsandbytes_4bit:
865
orig_qkv_offsets = {
866
"q": (0, self.num_heads * self.head_size),
867
"k": (self.num_heads * self.head_size,
@@ -871,7 +873,7 @@ class QKVParallelLinear(ColumnParallelLinear):
873
((self.num_heads + 2 * self.num_kv_heads) * self.head_size,
874
0)
875
}
874
- shard_size, shard_offset = adjust_bitsandbytes_shard(
876
+ shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(
877
param, orig_qkv_offsets, loaded_shard_id)
878
879
if is_gguf_weight:
vllm/model_executor/layers/quantization/bitsandbytes.py
+195
-36
@@ -1,7 +1,6 @@
1
from typing import Any, Dict, List, Optional
2
3
import torch
4
-from torch.nn.parameter import Parameter
4
5
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
6
set_weight_attrs)
@@ -15,8 +14,28 @@ class BitsAndBytesConfig(QuantizationConfig):
14
Reference: https://arxiv.org/abs/2305.14314
15
"""
16
18
- def __init__(self, ) -> None:
19
- pass
17
+ def __init__(
18
+ self,
19
+ load_in_8bit: bool = False,
20
+ load_in_4bit: bool = True,
21
+ bnb_4bit_compute_dtype: str = "float32",
22
+ bnb_4bit_quant_type: str = "fp4",
23
+ bnb_4bit_use_double_quant: bool = False,
24
+ llm_int8_enable_fp32_cpu_offload: bool = False,
25
+ llm_int8_has_fp16_weight: bool = False,
26
+ llm_int8_skip_modules: Optional[Any] = None,
27
+ llm_int8_threshold: float = 0.0,
28
+ ) -> None:
29
+
30
+ self.load_in_8bit = load_in_8bit
31
+ self.load_in_4bit = load_in_4bit
32
+ self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype
33
+ self.bnb_4bit_quant_type = bnb_4bit_quant_type
34
+ self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant
35
+ self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload
36
+ self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight
37
+ self.llm_int8_skip_modules = llm_int8_skip_modules
38
+ self.llm_int8_threshold = llm_int8_threshold
39
40
def __repr__(self) -> str:
41
return "BitsAndBytesConfig"
@@ -41,7 +60,46 @@ class BitsAndBytesConfig(QuantizationConfig):
60
61
@classmethod
62
def from_config(cls, config: Dict[str, Any]) -> "BitsAndBytesConfig":
44
- return cls()
63
+
64
+ def get_safe_value(config, keys, default_value=None):
65
+ try:
66
+ value = cls.get_from_keys(config, keys)
67
+ return value if value is not None else default_value
68
+ except ValueError:
69
+ return default_value
70
+
71
+ load_in_8bit = get_safe_value(config, ["load_in_8bit"],
72
+ default_value=False)
73
+ load_in_4bit = get_safe_value(config, ["load_in_4bit"],
74
+ default_value=True)
75
+ bnb_4bit_compute_dtype = get_safe_value(config,
76
+ ["bnb_4bit_compute_dtype"],
77
+ default_value="float32")
78
+ bnb_4bit_quant_type = get_safe_value(config, ["bnb_4bit_quant_type"],
79
+ default_value="fp4")
80
+ bnb_4bit_use_double_quant = get_safe_value(
81
+ config, ["bnb_4bit_use_double_quant"], default_value=False)
82
+ llm_int8_enable_fp32_cpu_offload = get_safe_value(
83
+ config, ["llm_int8_enable_fp32_cpu_offload"], default_value=False)
84
+ llm_int8_has_fp16_weight = get_safe_value(config,
85
+ ["llm_int8_has_fp16_weight"],
86
+ default_value=False)
87
+ llm_int8_skip_modules = get_safe_value(config,
88
+ ["llm_int8_skip_modules"],
89
+ default_value=[])
90
+ llm_int8_threshold = get_safe_value(config, ["llm_int8_threshold"],
91
+ default_value=0.0)
92
+
93
+ return cls(
94
+ load_in_8bit=load_in_8bit,
95
+ load_in_4bit=load_in_4bit,
96
+ bnb_4bit_compute_dtype=bnb_4bit_compute_dtype,
97
+ bnb_4bit_quant_type=bnb_4bit_quant_type,
98
+ bnb_4bit_use_double_quant=bnb_4bit_use_double_quant,
99
+ llm_int8_enable_fp32_cpu_offload=llm_int8_enable_fp32_cpu_offload,
100
+ llm_int8_has_fp16_weight=llm_int8_has_fp16_weight,
101
+ llm_int8_skip_modules=llm_int8_skip_modules,
102
+ llm_int8_threshold=llm_int8_threshold)
103
104
def get_quant_method(self, layer: torch.nn.Module,
105
prefix: str) -> Optional["BitsAndBytesLinearMethod"]:
@@ -78,39 +136,58 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
136
output_partition_sizes: List[int], input_size: int,
137
output_size: int, params_dtype: torch.dtype,
138
**extra_weight_attrs):
81
- quant_ratio = 0
82
- if params_dtype.is_floating_point:
83
- quant_ratio = torch.finfo(params_dtype).bits // torch.iinfo(
84
- torch.uint8).bits
139
+ from bitsandbytes.nn import Int8Params
140
+
141
+ def calculate_quant_ratio(dtype):
142
+ if dtype.is_floating_point:
143
+ return torch.finfo(dtype).bits // torch.iinfo(torch.uint8).bits
144
+ else:
145
+ return torch.iinfo(dtype).bits // torch.iinfo(torch.uint8).bits
146
+
147
+ def create_qweight_for_8bit():
148
+ qweight = Int8Params(
149
+ data=torch.empty(sum(output_partition_sizes),
150
+ input_size_per_partition,
151
+ dtype=torch.int8),
152
+ has_fp16_weights=self.quant_config.llm_int8_has_fp16_weight,
153
+ requires_grad=False)
154
+ set_weight_attrs(
155
+ qweight, {
156
+ "input_dim": 0,
157
+ "output_dim": 0,
158
+ "pack_factor": 1,
159
+ "use_bitsandbytes_8bit": True,
160
+ "generation": 0
161
+ })
162
+ return qweight
163
+
164
+ def create_qweight_for_4bit():
165
+ quant_ratio = calculate_quant_ratio(params_dtype)
166
+
167
+ total_size = input_size_per_partition * sum(output_partition_sizes)
168
+ if total_size % quant_ratio != 0:
169
+ raise ValueError(
170
+ "The input size is not aligned with the quantized "
171
+ "weight shape.")
172
+
173
+ qweight = torch.nn.Parameter(torch.empty(total_size // quant_ratio,
174
+ 1,
175
+ dtype=torch.uint8),
176
+ requires_grad=False)
177
+ set_weight_attrs(
178
+ qweight, {
179
+ "input_dim": 0,
180
+ "output_dim": 0,
181
+ "pack_factor": quant_ratio,
182
+ "use_bitsandbytes_4bit": True
183
+ })
184
+ return qweight
185
+
186
+ if self.quant_config.load_in_8bit:
187
+ qweight = create_qweight_for_8bit()
188
else:
86
- quant_ratio = torch.iinfo(params_dtype).bits // torch.iinfo(
87
- torch.uint8).bits
88
-
89
- if input_size_per_partition * sum(
90
- output_partition_sizes) % quant_ratio != 0:
91
- raise ValueError(
92
- "The input size is not aligned with the quantized "
93
- "weight shape. ")
94
- qweight = Parameter(
95
- torch.empty(
96
- input_size_per_partition * sum(output_partition_sizes) //
97
- quant_ratio,
98
- 1,
99
- dtype=torch.uint8,
100
- ),
101
- requires_grad=False,
102
- )
103
-
104
- set_weight_attrs(
105
- qweight,
106
- {
107
- "input_dim": 0,
108
- # In bitsandbytes, a tensor of shape [n,m] is quantized to
109
- #[n*m/pack_ratio, 1],so the output_dim is 0
110
- "output_dim": 0,
111
- "pack_factor": quant_ratio,
112
- "use_bitsandbytes": True,
113
- })
189
+ qweight = create_qweight_for_4bit()
190
+
191
layer.register_parameter("qweight", qweight)
192
set_weight_attrs(qweight, extra_weight_attrs)
193
@@ -119,6 +196,88 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
196
x: torch.Tensor,
197
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
198
199
+ if self.quant_config.load_in_8bit:
200
+ return self._apply_8bit_weight(layer, x, bias)
201
+ else:
202
+ return self._apply_4bit_weight(layer, x, bias)
203
+
204
+ def _apply_8bit_weight(
205
+ self,
206
+ layer: torch.nn.Module,
207
+ x: torch.Tensor,
208
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
209
+
210
+ # only load the bitsandbytes module when needed
211
+ from bitsandbytes import MatmulLtState, matmul
212
+
213
+ original_type = x.dtype
214
+ bf_x = x.to(torch.bfloat16)
215
+
216
+ qweight = layer.qweight
217
+ offsets = qweight.bnb_shard_offsets
218
+ quant_states = qweight.bnb_quant_state
219
+ matmul_states = qweight.matmul_state
220
+ generation = qweight.generation
221
+
222
+ out_dim_0 = x.shape[0]
223
+ out_dim_1 = sum(
224
+ [quant_state[1].shape[0] for quant_state in quant_states.items()])
225
+ out = torch.empty(out_dim_0,
226
+ out_dim_1,
227
+ dtype=torch.float16,
228
+ device=x.device)
229
+
230
+ current_index = 0
231
+ for i in range(len(quant_states)):
232
+ output_size = quant_states[i].shape[0]
233
+
234
+ # in profile_run or the first generation of inference,
235
+ # create new matmul_states
236
+ if generation == 0 or generation == 1:
237
+ matmul_states[i] = MatmulLtState()
238
+ matmul_states[i].CB = qweight[offsets[i]:offsets[i + 1]]
239
+ matmul_states[i].SCB = quant_states[i]
240
+ matmul_states[i].threshold = (
241
+ self.quant_config.llm_int8_threshold)
242
+ matmul_states[i].has_fp16_weights = (
243
+ self.quant_config.llm_int8_has_fp16_weight)
244
+ matmul_states[i].is_training = False
245
+ if matmul_states[i].threshold > 0.0 and not matmul_states[
246
+ i].has_fp16_weights:
247
+ matmul_states[i].use_pool = True
248
+
249
+ new_x = bf_x.unsqueeze(0)
250
+
251
+ out[:, current_index:current_index + output_size] = matmul(
252
+ new_x,
253
+ qweight[offsets[i]:offsets[i + 1]],
254
+ state=matmul_states[i])
255
+
256
+ current_index += output_size
257
+
258
+ # only update the matmul_states if it is not profile_run
259
+ if (generation > 0
260
+ and not self.quant_config.llm_int8_has_fp16_weight
261
+ and matmul_states[i].CB is not None
262
+ and matmul_states[i].CxB is not None):
263
+ del matmul_states[i].CB
264
+ qweight[offsets[i]:offsets[i + 1]] = matmul_states[i].CxB
265
+
266
+ out = out.to(original_type)
267
+
268
+ if bias is not None:
269
+ out += bias
270
+
271
+ qweight.generation += 1
272
+
273
+ return out
274
+
275
+ def _apply_4bit_weight(
276
+ self,
277
+ layer: torch.nn.Module,
278
+ x: torch.Tensor,
279
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
280
+
281
# only load the bitsandbytes module when needed
282
from bitsandbytes import matmul_4bit
283
vllm/model_executor/model_loader/loader.py
+126
-79
@@ -771,7 +771,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
771
return pt_weights_iterator(hf_weights_files)
772
773
def _get_quantized_weights_iterator(
774
- self, model_name_or_path: str, revision: Optional[str], pre_quant: bool
774
+ self,
775
+ model_name_or_path: str,
776
+ revision: Optional[str],
777
+ pre_quant: bool,
778
+ load_8bit: bool,
779
) -> Tuple[Generator[Tuple[str, torch.Tensor], None, None], Dict[str,
780
Any]]:
781
"""Get an iterator to the model weights with bitsandbytes quantization,
@@ -780,11 +784,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
784
# only load the bitsandbytes module when needed
785
try:
786
import bitsandbytes
783
- from bitsandbytes.functional import QuantState
787
if bitsandbytes.__version__ < "0.42.0":
788
raise ImportError("bitsandbytes version is wrong. Please "
789
"install bitsandbytes>=0.42.0.")
787
- from bitsandbytes.functional import quantize_4bit
790
except ImportError as err:
791
raise ImportError("Please install bitsandbytes>=0.42.0 via "
792
"`pip install bitsandbytes>=0.42.0` to use "
@@ -793,80 +795,111 @@ class BitsAndBytesModelLoader(BaseModelLoader):
795
hf_weights_files, use_safetensors = self._prepare_weights(
796
model_name_or_path, revision)
797
796
- quant_state_dict = {}
797
-
798
- def quantized_checkpoint() -> Generator:
799
- # First iterate over all quant state weights
800
- weight_iterator = self._hf_weight_iter(hf_weights_files,
801
- use_safetensors)
802
- temp_state_dict = {}
803
- for weight_name, weight_tensor in weight_iterator:
804
- if weight_name.endswith(".weight"):
805
- continue
806
- # TODO: only nf4 quantization is supported for now
807
- if weight_name.endswith(".quant_state.bitsandbytes__fp4"):
808
- raise NotImplementedError(
809
- "Only bitsandbytes_nf4 quantization"
810
- f"is supported for now. {weight_name} is fp4 quantized"
811
- )
812
- temp_state_dict[weight_name] = weight_tensor
798
+ quant_state_dict: Dict[str, Any] = {}
799
814
- # Closure to parse quant_state for each prequant weight
815
- def _parse_quant_state(param_name: str,
816
- temp_state_dict: Dict) -> QuantState:
817
- quant_state = {}
818
- for k in temp_state_dict:
819
- if param_name + "." in k:
820
- quant_state[k] = temp_state_dict[k]
821
- # bitsandbytes library requires
822
- # weight.quant_state.bitsandbytes__nf4 in CPU
823
- quant_state[param_name +
824
- ".quant_state.bitsandbytes__nf4"] = quant_state[
825
- param_name +
826
- ".quant_state.bitsandbytes__nf4"].cpu().data
827
- return QuantState.from_dict(quant_state, device="cuda")
828
-
829
- # Second iterate over all prequant and normal weights
830
- # pre quantized weights would have a quant_state
831
- for weight_name, weight_tensor in self._hf_weight_iter(
832
- hf_weights_files, use_safetensors):
833
- # Filter out all weights whose suffix is not ".weight"
834
- if not weight_name.endswith(".weight"):
835
- continue
836
- if weight_name + ".quant_state.bitsandbytes__nf4" \
837
- in temp_state_dict:
838
- quant_state = _parse_quant_state(weight_name,
839
- temp_state_dict)
840
- weight_name = weight_name.replace(".weight", ".qweight")
841
- quant_state_dict[weight_name] = quant_state
842
- yield weight_name.replace(".weight",
843
- ".qweight"), weight_tensor
844
- else:
845
- yield weight_name, weight_tensor
846
-
847
- def generator() -> Generator:
848
- for weight_name, weight_tensor in self._hf_weight_iter(
849
- hf_weights_files, use_safetensors):
850
- if any(target_module in weight_name
851
- for target_module in self.target_modules):
852
- weight_name = weight_name.replace(".weight", ".qweight")
853
- # bitsandbytes requires data in GPU
854
- loaded_weight = weight_tensor.cuda().data
855
- with set_default_torch_dtype(torch.float32):
856
- processed_weight, quant_state = quantize_4bit(
857
- loaded_weight,
858
- compress_statistics=True,
859
- quant_type="nf4")
860
-
861
- quant_state_dict[weight_name] = quant_state
862
- else:
863
- processed_weight = weight_tensor
800
+ if pre_quant:
801
+ if load_8bit:
802
+ return self._quantized_8bit_generator(
803
+ hf_weights_files, use_safetensors,
804
+ quant_state_dict), quant_state_dict
805
+ else:
806
+ return self._quantized_4bit_generator(
807
+ hf_weights_files, use_safetensors,
808
+ quant_state_dict), quant_state_dict
809
865
- yield weight_name, processed_weight
810
+ return self._unquantized_generator(hf_weights_files, use_safetensors,
811
+ quant_state_dict), quant_state_dict
812
867
- if pre_quant:
868
- return quantized_checkpoint(), quant_state_dict
869
- return generator(), quant_state_dict
813
+ def _quantized_8bit_generator(self, hf_weights_files, use_safetensors,
814
+ quant_state_dict) -> Generator:
815
+ for weight_name, weight_tensor in self._hf_weight_iter(
816
+ hf_weights_files, use_safetensors):
817
+ if not weight_name.lower().endswith(".scb"):
818
+ continue
819
+
820
+ weight_key = weight_name.lower().replace(".scb", ".qweight")
821
+ quant_state_dict[weight_key] = weight_tensor
822
+
823
+ for weight_name, weight_tensor in self._hf_weight_iter(
824
+ hf_weights_files, use_safetensors):
825
+
826
+ if not weight_name.endswith(".weight"):
827
+ continue
828
+
829
+ qweight_name = weight_name.replace(".weight", ".qweight")
830
+ if qweight_name in quant_state_dict:
831
+ set_weight_attrs(weight_tensor, {"load_in_8bit": True})
832
+ yield qweight_name, weight_tensor
833
+ else:
834
+ yield weight_name, weight_tensor
835
+
836
+ def _quantized_4bit_generator(self, hf_weights_files, use_safetensors,
837
+ quant_state_dict) -> Generator:
838
+ from bitsandbytes.functional import QuantState
839
+
840
+ # First iterate over all quant state weights
841
+ weight_iterator = self._hf_weight_iter(hf_weights_files,
842
+ use_safetensors)
843
+ temp_state_dict = {}
844
+ for weight_name, weight_tensor in weight_iterator:
845
+ if weight_name.endswith(".weight"):
846
+ continue
847
+ # bitsandbytes library requires
848
+ # weight.quant_state.bitsandbytes__* in CPU
849
+ if "quant_state.bitsandbytes" in weight_name:
850
+ temp_state_dict[weight_name] = weight_tensor.cpu().data
851
+ else:
852
+ temp_state_dict[weight_name] = weight_tensor
853
+
854
+ # Closure to parse quant_state for each prequant weight
855
+ def _parse_quant_state(param_name: str,
856
+ temp_state_dict: Dict) -> QuantState:
857
+ quant_state = {}
858
+ for k in temp_state_dict:
859
+ if param_name + "." in k:
860
+ quant_state[k] = temp_state_dict[k]
861
+
862
+ return QuantState.from_dict(quant_state, device="cuda")
863
+
864
+ # Second iterate over all prequant and normal weights
865
+ # pre quantized weights would have a quant_state
866
+ for weight_name, weight_tensor in self._hf_weight_iter(
867
+ hf_weights_files, use_safetensors):
868
+ # Filter out all weights whose suffix is not ".weight"
869
+ if not weight_name.endswith(".weight"):
870
+ continue
871
+ if (f"{weight_name}.quant_state.bitsandbytes__nf4" \
872
+ in temp_state_dict) or \
873
+ (f"{weight_name}.quant_state.bitsandbytes__fp4" \
874
+ in temp_state_dict):
875
+ quant_state = _parse_quant_state(weight_name, temp_state_dict)
876
+ weight_name = weight_name.replace(".weight", ".qweight")
877
+ quant_state_dict[weight_name] = quant_state
878
+ yield weight_name.replace(".weight", ".qweight"), weight_tensor
879
+ else:
880
+ yield weight_name, weight_tensor
881
+
882
+ def _unquantized_generator(self, hf_weights_files, use_safetensors,
883
+ quant_state_dict) -> Generator:
884
+ from bitsandbytes.functional import quantize_4bit
885
+ for weight_name, weight_tensor in self._hf_weight_iter(
886
+ hf_weights_files, use_safetensors):
887
+ if any(target_module in weight_name
888
+ for target_module in self.target_modules):
889
+ weight_name = weight_name.replace(".weight", ".qweight")
890
+ # bitsandbytes requires data in GPU
891
+ loaded_weight = weight_tensor.cuda().data
892
+ with set_default_torch_dtype(torch.float32):
893
+ processed_weight, quant_state = quantize_4bit(
894
+ loaded_weight,
895
+ compress_statistics=True,
896
+ quant_type="nf4")
897
+
898
+ quant_state_dict[weight_name] = quant_state
899
+ else:
900
+ processed_weight = weight_tensor
901
+
902
+ yield weight_name, processed_weight
903
904
def _load_weights(self, model_config: ModelConfig,
905
model: nn.Module) -> None:
@@ -883,16 +916,26 @@ class BitsAndBytesModelLoader(BaseModelLoader):
916
logger.info("Loading weights with BitsAndBytes quantization. "
917
" May take a while ...")
918
886
- is_quantized_checkpoint = False
919
quant_config = getattr(model_config.hf_config, "quantization_config",
920
None)
889
- if quant_config is not None and quant_config.get(
890
- 'quant_method') == "bitsandbytes":
891
- is_quantized_checkpoint = True
921
+
922
+ pre_quant = False
923
+ if quant_config is not None:
924
+ quant_method = quant_config.get('quant_method')
925
+ if quant_method == "bitsandbytes":
926
+ pre_quant = True
927
+ else:
928
+ raise ValueError(
929
+ f"BitsAndBytes loader does not support {quant_method} "
930
+ "quantization")
931
+
932
+ load_8bit = False
933
+ if pre_quant:
934
+ load_8bit = quant_config.get('load_in_8bit', False)
935
936
qweight_iterator, quant_state_dict = \
937
self._get_quantized_weights_iterator(
895
- model_config.model, model_config.revision, is_quantized_checkpoint)
938
+ model_config.model, model_config.revision, pre_quant, load_8bit)
939
940
model.load_weights(qweight_iterator)
941
@@ -942,6 +985,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
985
offsets = np.concatenate(([0], np.cumsum(num_elements)))
986
set_weight_attrs(param, {"bnb_shard_offsets": offsets})
987
988
+ if load_8bit:
989
+ set_weight_attrs(
990
+ param, {"matmul_state": [None] * len(quant_states)})
991
+
992
def load_model(self, *, model_config: ModelConfig,
993
device_config: DeviceConfig,
994
lora_config: Optional[LoRAConfig],