@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4664ceaad6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

support bitsandbytes 8-bit and FP4 quantized models (#7445)

chenqianfzh committed Aug 29, 2024 at 16:09 UTC 4664ceaad6f99ec7824859d1ac31b29502565a98
6 files changed +437 -191
tests/conftest.py
+6
@@ -209,8 +209,14 @@ class HfRunner:
209
210 def wrap_device(self, input: _T) -> _T:
211 if not is_cpu():
212 + # Check if the input is already on the GPU
213 + if hasattr(input, 'device') and input.device.type == "cuda":
214 + return input # Already on GPU, no need to move
215 return input.to("cuda")
216 else:
217 + # Check if the input is already on the CPU
218 + if hasattr(input, 'device') and input.device.type == "cpu":
219 + return input # Already on CPU, no need to move
220 return input.to("cpu")
221
222 def __init__(
tests/quantization/test_bitsandbytes.py
+98 -68
@@ -2,85 +2,115 @@
2
3 Run `pytest tests/quantization/test_bitsandbytes.py`.
4 '''
5 +
6 +import gc
7 +
8 import pytest
9 import torch
10
11 from tests.quantization.utils import is_quant_method_supported
9 -from vllm import SamplingParams
12
11 -models_to_test = [
13 +models_4bit_to_test = [
14 ('huggyllama/llama-7b', 'quantize model inflight'),
13 - ('lllyasviel/omost-llama-3-8b-4bits', 'read pre-quantized model'),
15 ]
16
17 +models_pre_qaunt_4bit_to_test = [
18 + ('lllyasviel/omost-llama-3-8b-4bits',
19 + 'read pre-quantized 4-bit NF4 model'),
20 + ('PrunaAI/Einstein-v6.1-Llama3-8B-bnb-4bit-smashed',
21 + 'read pre-quantized 4-bit FP4 model'),
22 +]
23 +
24 +models_pre_quant_8bit_to_test = [
25 + ('meta-llama/Llama-Guard-3-8B-INT8', 'read pre-quantized 8-bit model'),
26 +]
27 +
28 +
29 +@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
30 + reason='bitsandbytes is not supported on this GPU type.')
31 +@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
32 +def test_load_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
33 + model_name, description) -> None:
34 +
35 + hf_model_kwargs = {"load_in_4bit": True}
36 + validate_generated_texts(hf_runner, vllm_runner, example_prompts[:1],
37 + model_name, hf_model_kwargs)
38 +
39 +
40 +@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
41 + reason='bitsandbytes is not supported on this GPU type.')
42 +@pytest.mark.parametrize("model_name, description",
43 + models_pre_qaunt_4bit_to_test)
44 +def test_load_pre_quant_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
45 + model_name, description) -> None:
46 +
47 + validate_generated_texts(hf_runner, vllm_runner, example_prompts[:1],
48 + model_name)
49 +
50
51 @pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
52 reason='bitsandbytes is not supported on this GPU type.')
19 -@pytest.mark.parametrize("model_name, description", models_to_test)
20 -def test_load_bnb_model(vllm_runner, model_name, description) -> None:
53 +@pytest.mark.parametrize("model_name, description",
54 + models_pre_quant_8bit_to_test)
55 +def test_load_8bit_bnb_model(hf_runner, vllm_runner, example_prompts,
56 + model_name, description) -> None:
57 +
58 + validate_generated_texts(hf_runner, vllm_runner, example_prompts[:1],
59 + model_name)
60 +
61 +
62 +def log_generated_texts(prompts, outputs, runner_name):
63 + logged_texts = []
64 + for i, (_, generated_text) in enumerate(outputs):
65 + log_entry = {
66 + "prompt": prompts[i],
67 + "runner_name": runner_name,
68 + "generated_text": generated_text,
69 + }
70 + logged_texts.append(log_entry)
71 + return logged_texts
72 +
73 +
74 +def validate_generated_texts(hf_runner,
75 + vllm_runner,
76 + prompts,
77 + model_name,
78 + hf_model_kwargs=None):
79 +
80 + if hf_model_kwargs is None:
81 + hf_model_kwargs = {}
82 +
83 + # Run with HF runner
84 + with hf_runner(model_name, model_kwargs=hf_model_kwargs) as llm:
85 + hf_outputs = llm.generate_greedy(prompts, 8)
86 + hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")
87 +
88 + # Clean up the GPU memory for the next test
89 + torch.cuda.synchronize()
90 + gc.collect()
91 + torch.cuda.empty_cache()
92 +
93 + #Run with vLLM runner
94 with vllm_runner(model_name,
95 quantization='bitsandbytes',
96 load_format='bitsandbytes',
24 - enforce_eager=True) as llm:
25 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
26 -
27 - # check the weights in MLP & SelfAttention are quantized to torch.uint8
28 - qweight = model.model.layers[0].mlp.gate_up_proj.qweight
29 - assert qweight.dtype == torch.uint8, (
30 - f'Expected gate_up_proj dtype torch.uint8 but got {qweight.dtype}')
31 -
32 - qweight = model.model.layers[0].mlp.down_proj.qweight
33 - assert qweight.dtype == torch.uint8, (
34 - f'Expected down_proj dtype torch.uint8 but got {qweight.dtype}')
35 -
36 - qweight = model.model.layers[0].self_attn.o_proj.qweight
37 - assert qweight.dtype == torch.uint8, (
38 - f'Expected o_proj dtype torch.uint8 but got {qweight.dtype}')
39 -
40 - qweight = model.model.layers[0].self_attn.qkv_proj.qweight
41 - assert qweight.dtype == torch.uint8, (
42 - f'Expected qkv_proj dtype torch.uint8 but got {qweight.dtype}')
43 -
44 - # some weights should not be quantized
45 - weight = model.lm_head.weight
46 - assert weight.dtype != torch.uint8, (
47 - 'lm_head weight dtype should not be torch.uint8')
48 -
49 - weight = model.model.embed_tokens.weight
50 - assert weight.dtype != torch.uint8, (
51 - 'embed_tokens weight dtype should not be torch.uint8')
52 -
53 - weight = model.model.layers[0].input_layernorm.weight
54 - assert weight.dtype != torch.uint8, (
55 - 'input_layernorm weight dtype should not be torch.uint8')
56 -
57 - weight = model.model.layers[0].post_attention_layernorm.weight
58 - assert weight.dtype != torch.uint8, (
59 - 'input_layernorm weight dtype should not be torch.uint8')
60 -
61 - # check the output of the model is expected
62 - sampling_params = SamplingParams(temperature=0.0,
63 - logprobs=1,
64 - prompt_logprobs=1,
65 - max_tokens=8)
66 -
67 - prompts = ['That which does not kill us', 'To be or not to be,']
68 - expected_outputs = [
69 - 'That which does not kill us makes us stronger.',
70 - 'To be or not to be, that is the question.'
71 - ]
72 - outputs = llm.generate(prompts, sampling_params=sampling_params)
73 - assert len(outputs) == len(prompts)
74 -
75 - for index in range(len(outputs)):
76 - # compare the first line of the output
77 - actual_output = outputs[index][1][0].split('\n', 1)[0]
78 - expected_output = expected_outputs[index].split('\n', 1)[0]
79 -
80 - assert len(actual_output) >= len(expected_output), (
81 - f'Actual {actual_output} should be larger than or equal to '
82 - f'expected {expected_output}')
83 - actual_output = actual_output[:len(expected_output)]
84 -
85 - assert actual_output == expected_output, (
86 - f'Expected: {expected_output}, but got: {actual_output}')
97 + enforce_eager=True,
98 + gpu_memory_utilization=0.8) as llm:
99 + vllm_outputs = llm.generate_greedy(prompts, 8)
100 + vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")
101 +
102 + # Clean up the GPU memory for the next test
103 + torch.cuda.synchronize()
104 + gc.collect()
105 + torch.cuda.empty_cache()
106 +
107 + # Compare the generated strings
108 + for hf_log, vllm_log in zip(hf_logs, vllm_logs):
109 + hf_str = hf_log["generated_text"]
110 + vllm_str = vllm_log["generated_text"]
111 + prompt = hf_log["prompt"]
112 + assert hf_str == vllm_str, (f"Model: {model_name}"
113 + f"Mismatch between HF and vLLM outputs:\n"
114 + f"Prompt: {prompt}\n"
115 + f"HF Output: '{hf_str}'\n"
116 + f"vLLM Output: '{vllm_str}'")
vllm/config.py
+2
@@ -405,6 +405,8 @@ class ModelConfig:
405 raise ValueError(
406 "BitAndBytes quantization with TP or PP is not supported yet.")
407
408 + # Remove the constraint after the bitsandbytes issue is fixed:
409 + # https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1308
410 if self.quantization == "bitsandbytes" and self.enforce_eager is False:
411 logger.warning("CUDA graph is not supported on BitAndBytes yet, "
412 "fallback to the eager mode.")
vllm/model_executor/layers/linear.py
+10 -8
@@ -36,9 +36,9 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
36 return shard_size * marlin_tile_size, shard_offset * marlin_tile_size
37
38
39 -def adjust_bitsandbytes_shard(param: Parameter,
40 - qkv_offsets: Dict[str, Tuple[int, int]],
41 - loaded_shard_id: str) -> Tuple[int, int]:
39 +def adjust_bitsandbytes_4bit_shard(param: Parameter,
40 + qkv_offsets: Dict[str, Tuple[int, int]],
41 + loaded_shard_id: str) -> Tuple[int, int]:
42 """Adjust the quantization offsets and sizes for BitsAndBytes sharding."""
43
44 total, _ = qkv_offsets["total"]
@@ -505,8 +505,9 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
505 shard_size, shard_offset = adjust_marlin_shard(
506 param, shard_size, shard_offset)
507
508 - use_bitsandbytes = getattr(param, "use_bitsandbytes", False)
509 - if use_bitsandbytes:
508 + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
509 + False)
510 + if use_bitsandbytes_4bit:
511 shard_size = loaded_weight.shape[output_dim]
512 shard_offset = loaded_weight.shape[output_dim] * \
513 loaded_shard_id
@@ -858,8 +859,9 @@ class QKVParallelLinear(ColumnParallelLinear):
859 shard_size, shard_offset = adjust_marlin_shard(
860 param, shard_size, shard_offset)
861
861 - use_bitsandbytes = getattr(param, "use_bitsandbytes", False)
862 - if use_bitsandbytes:
862 + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
863 + False)
864 + if use_bitsandbytes_4bit:
865 orig_qkv_offsets = {
866 "q": (0, self.num_heads * self.head_size),
867 "k": (self.num_heads * self.head_size,
@@ -871,7 +873,7 @@ class QKVParallelLinear(ColumnParallelLinear):
873 ((self.num_heads + 2 * self.num_kv_heads) * self.head_size,
874 0)
875 }
874 - shard_size, shard_offset = adjust_bitsandbytes_shard(
876 + shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(
877 param, orig_qkv_offsets, loaded_shard_id)
878
879 if is_gguf_weight:
vllm/model_executor/layers/quantization/bitsandbytes.py
+195 -36
@@ -1,7 +1,6 @@
1 from typing import Any, Dict, List, Optional
2
3 import torch
4 -from torch.nn.parameter import Parameter
4
5 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
6 set_weight_attrs)
@@ -15,8 +14,28 @@ class BitsAndBytesConfig(QuantizationConfig):
14 Reference: https://arxiv.org/abs/2305.14314
15 """
16
18 - def __init__(self, ) -> None:
19 - pass
17 + def __init__(
18 + self,
19 + load_in_8bit: bool = False,
20 + load_in_4bit: bool = True,
21 + bnb_4bit_compute_dtype: str = "float32",
22 + bnb_4bit_quant_type: str = "fp4",
23 + bnb_4bit_use_double_quant: bool = False,
24 + llm_int8_enable_fp32_cpu_offload: bool = False,
25 + llm_int8_has_fp16_weight: bool = False,
26 + llm_int8_skip_modules: Optional[Any] = None,
27 + llm_int8_threshold: float = 0.0,
28 + ) -> None:
29 +
30 + self.load_in_8bit = load_in_8bit
31 + self.load_in_4bit = load_in_4bit
32 + self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype
33 + self.bnb_4bit_quant_type = bnb_4bit_quant_type
34 + self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant
35 + self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload
36 + self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight
37 + self.llm_int8_skip_modules = llm_int8_skip_modules
38 + self.llm_int8_threshold = llm_int8_threshold
39
40 def __repr__(self) -> str:
41 return "BitsAndBytesConfig"
@@ -41,7 +60,46 @@ class BitsAndBytesConfig(QuantizationConfig):
60
61 @classmethod
62 def from_config(cls, config: Dict[str, Any]) -> "BitsAndBytesConfig":
44 - return cls()
63 +
64 + def get_safe_value(config, keys, default_value=None):
65 + try:
66 + value = cls.get_from_keys(config, keys)
67 + return value if value is not None else default_value
68 + except ValueError:
69 + return default_value
70 +
71 + load_in_8bit = get_safe_value(config, ["load_in_8bit"],
72 + default_value=False)
73 + load_in_4bit = get_safe_value(config, ["load_in_4bit"],
74 + default_value=True)
75 + bnb_4bit_compute_dtype = get_safe_value(config,
76 + ["bnb_4bit_compute_dtype"],
77 + default_value="float32")
78 + bnb_4bit_quant_type = get_safe_value(config, ["bnb_4bit_quant_type"],
79 + default_value="fp4")
80 + bnb_4bit_use_double_quant = get_safe_value(
81 + config, ["bnb_4bit_use_double_quant"], default_value=False)
82 + llm_int8_enable_fp32_cpu_offload = get_safe_value(
83 + config, ["llm_int8_enable_fp32_cpu_offload"], default_value=False)
84 + llm_int8_has_fp16_weight = get_safe_value(config,
85 + ["llm_int8_has_fp16_weight"],
86 + default_value=False)
87 + llm_int8_skip_modules = get_safe_value(config,
88 + ["llm_int8_skip_modules"],
89 + default_value=[])
90 + llm_int8_threshold = get_safe_value(config, ["llm_int8_threshold"],
91 + default_value=0.0)
92 +
93 + return cls(
94 + load_in_8bit=load_in_8bit,
95 + load_in_4bit=load_in_4bit,
96 + bnb_4bit_compute_dtype=bnb_4bit_compute_dtype,
97 + bnb_4bit_quant_type=bnb_4bit_quant_type,
98 + bnb_4bit_use_double_quant=bnb_4bit_use_double_quant,
99 + llm_int8_enable_fp32_cpu_offload=llm_int8_enable_fp32_cpu_offload,
100 + llm_int8_has_fp16_weight=llm_int8_has_fp16_weight,
101 + llm_int8_skip_modules=llm_int8_skip_modules,
102 + llm_int8_threshold=llm_int8_threshold)
103
104 def get_quant_method(self, layer: torch.nn.Module,
105 prefix: str) -> Optional["BitsAndBytesLinearMethod"]:
@@ -78,39 +136,58 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
136 output_partition_sizes: List[int], input_size: int,
137 output_size: int, params_dtype: torch.dtype,
138 **extra_weight_attrs):
81 - quant_ratio = 0
82 - if params_dtype.is_floating_point:
83 - quant_ratio = torch.finfo(params_dtype).bits // torch.iinfo(
84 - torch.uint8).bits
139 + from bitsandbytes.nn import Int8Params
140 +
141 + def calculate_quant_ratio(dtype):
142 + if dtype.is_floating_point:
143 + return torch.finfo(dtype).bits // torch.iinfo(torch.uint8).bits
144 + else:
145 + return torch.iinfo(dtype).bits // torch.iinfo(torch.uint8).bits
146 +
147 + def create_qweight_for_8bit():
148 + qweight = Int8Params(
149 + data=torch.empty(sum(output_partition_sizes),
150 + input_size_per_partition,
151 + dtype=torch.int8),
152 + has_fp16_weights=self.quant_config.llm_int8_has_fp16_weight,
153 + requires_grad=False)
154 + set_weight_attrs(
155 + qweight, {
156 + "input_dim": 0,
157 + "output_dim": 0,
158 + "pack_factor": 1,
159 + "use_bitsandbytes_8bit": True,
160 + "generation": 0
161 + })
162 + return qweight
163 +
164 + def create_qweight_for_4bit():
165 + quant_ratio = calculate_quant_ratio(params_dtype)
166 +
167 + total_size = input_size_per_partition * sum(output_partition_sizes)
168 + if total_size % quant_ratio != 0:
169 + raise ValueError(
170 + "The input size is not aligned with the quantized "
171 + "weight shape.")
172 +
173 + qweight = torch.nn.Parameter(torch.empty(total_size // quant_ratio,
174 + 1,
175 + dtype=torch.uint8),
176 + requires_grad=False)
177 + set_weight_attrs(
178 + qweight, {
179 + "input_dim": 0,
180 + "output_dim": 0,
181 + "pack_factor": quant_ratio,
182 + "use_bitsandbytes_4bit": True
183 + })
184 + return qweight
185 +
186 + if self.quant_config.load_in_8bit:
187 + qweight = create_qweight_for_8bit()
188 else:
86 - quant_ratio = torch.iinfo(params_dtype).bits // torch.iinfo(
87 - torch.uint8).bits
88 -
89 - if input_size_per_partition * sum(
90 - output_partition_sizes) % quant_ratio != 0:
91 - raise ValueError(
92 - "The input size is not aligned with the quantized "
93 - "weight shape. ")
94 - qweight = Parameter(
95 - torch.empty(
96 - input_size_per_partition * sum(output_partition_sizes) //
97 - quant_ratio,
98 - 1,
99 - dtype=torch.uint8,
100 - ),
101 - requires_grad=False,
102 - )
103 -
104 - set_weight_attrs(
105 - qweight,
106 - {
107 - "input_dim": 0,
108 - # In bitsandbytes, a tensor of shape [n,m] is quantized to
109 - #[n*m/pack_ratio, 1],so the output_dim is 0
110 - "output_dim": 0,
111 - "pack_factor": quant_ratio,
112 - "use_bitsandbytes": True,
113 - })
189 + qweight = create_qweight_for_4bit()
190 +
191 layer.register_parameter("qweight", qweight)
192 set_weight_attrs(qweight, extra_weight_attrs)
193
@@ -119,6 +196,88 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
196 x: torch.Tensor,
197 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
198
199 + if self.quant_config.load_in_8bit:
200 + return self._apply_8bit_weight(layer, x, bias)
201 + else:
202 + return self._apply_4bit_weight(layer, x, bias)
203 +
204 + def _apply_8bit_weight(
205 + self,
206 + layer: torch.nn.Module,
207 + x: torch.Tensor,
208 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
209 +
210 + # only load the bitsandbytes module when needed
211 + from bitsandbytes import MatmulLtState, matmul
212 +
213 + original_type = x.dtype
214 + bf_x = x.to(torch.bfloat16)
215 +
216 + qweight = layer.qweight
217 + offsets = qweight.bnb_shard_offsets
218 + quant_states = qweight.bnb_quant_state
219 + matmul_states = qweight.matmul_state
220 + generation = qweight.generation
221 +
222 + out_dim_0 = x.shape[0]
223 + out_dim_1 = sum(
224 + [quant_state[1].shape[0] for quant_state in quant_states.items()])
225 + out = torch.empty(out_dim_0,
226 + out_dim_1,
227 + dtype=torch.float16,
228 + device=x.device)
229 +
230 + current_index = 0
231 + for i in range(len(quant_states)):
232 + output_size = quant_states[i].shape[0]
233 +
234 + # in profile_run or the first generation of inference,
235 + # create new matmul_states
236 + if generation == 0 or generation == 1:
237 + matmul_states[i] = MatmulLtState()
238 + matmul_states[i].CB = qweight[offsets[i]:offsets[i + 1]]
239 + matmul_states[i].SCB = quant_states[i]
240 + matmul_states[i].threshold = (
241 + self.quant_config.llm_int8_threshold)
242 + matmul_states[i].has_fp16_weights = (
243 + self.quant_config.llm_int8_has_fp16_weight)
244 + matmul_states[i].is_training = False
245 + if matmul_states[i].threshold > 0.0 and not matmul_states[
246 + i].has_fp16_weights:
247 + matmul_states[i].use_pool = True
248 +
249 + new_x = bf_x.unsqueeze(0)
250 +
251 + out[:, current_index:current_index + output_size] = matmul(
252 + new_x,
253 + qweight[offsets[i]:offsets[i + 1]],
254 + state=matmul_states[i])
255 +
256 + current_index += output_size
257 +
258 + # only update the matmul_states if it is not profile_run
259 + if (generation > 0
260 + and not self.quant_config.llm_int8_has_fp16_weight
261 + and matmul_states[i].CB is not None
262 + and matmul_states[i].CxB is not None):
263 + del matmul_states[i].CB
264 + qweight[offsets[i]:offsets[i + 1]] = matmul_states[i].CxB
265 +
266 + out = out.to(original_type)
267 +
268 + if bias is not None:
269 + out += bias
270 +
271 + qweight.generation += 1
272 +
273 + return out
274 +
275 + def _apply_4bit_weight(
276 + self,
277 + layer: torch.nn.Module,
278 + x: torch.Tensor,
279 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
280 +
281 # only load the bitsandbytes module when needed
282 from bitsandbytes import matmul_4bit
283
vllm/model_executor/model_loader/loader.py
+126 -79
@@ -771,7 +771,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
771 return pt_weights_iterator(hf_weights_files)
772
773 def _get_quantized_weights_iterator(
774 - self, model_name_or_path: str, revision: Optional[str], pre_quant: bool
774 + self,
775 + model_name_or_path: str,
776 + revision: Optional[str],
777 + pre_quant: bool,
778 + load_8bit: bool,
779 ) -> Tuple[Generator[Tuple[str, torch.Tensor], None, None], Dict[str,
780 Any]]:
781 """Get an iterator to the model weights with bitsandbytes quantization,
@@ -780,11 +784,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
784 # only load the bitsandbytes module when needed
785 try:
786 import bitsandbytes
783 - from bitsandbytes.functional import QuantState
787 if bitsandbytes.__version__ < "0.42.0":
788 raise ImportError("bitsandbytes version is wrong. Please "
789 "install bitsandbytes>=0.42.0.")
787 - from bitsandbytes.functional import quantize_4bit
790 except ImportError as err:
791 raise ImportError("Please install bitsandbytes>=0.42.0 via "
792 "`pip install bitsandbytes>=0.42.0` to use "
@@ -793,80 +795,111 @@ class BitsAndBytesModelLoader(BaseModelLoader):
795 hf_weights_files, use_safetensors = self._prepare_weights(
796 model_name_or_path, revision)
797
796 - quant_state_dict = {}
797 -
798 - def quantized_checkpoint() -> Generator:
799 - # First iterate over all quant state weights
800 - weight_iterator = self._hf_weight_iter(hf_weights_files,
801 - use_safetensors)
802 - temp_state_dict = {}
803 - for weight_name, weight_tensor in weight_iterator:
804 - if weight_name.endswith(".weight"):
805 - continue
806 - # TODO: only nf4 quantization is supported for now
807 - if weight_name.endswith(".quant_state.bitsandbytes__fp4"):
808 - raise NotImplementedError(
809 - "Only bitsandbytes_nf4 quantization"
810 - f"is supported for now. {weight_name} is fp4 quantized"
811 - )
812 - temp_state_dict[weight_name] = weight_tensor
798 + quant_state_dict: Dict[str, Any] = {}
799
814 - # Closure to parse quant_state for each prequant weight
815 - def _parse_quant_state(param_name: str,
816 - temp_state_dict: Dict) -> QuantState:
817 - quant_state = {}
818 - for k in temp_state_dict:
819 - if param_name + "." in k:
820 - quant_state[k] = temp_state_dict[k]
821 - # bitsandbytes library requires
822 - # weight.quant_state.bitsandbytes__nf4 in CPU
823 - quant_state[param_name +
824 - ".quant_state.bitsandbytes__nf4"] = quant_state[
825 - param_name +
826 - ".quant_state.bitsandbytes__nf4"].cpu().data
827 - return QuantState.from_dict(quant_state, device="cuda")
828 -
829 - # Second iterate over all prequant and normal weights
830 - # pre quantized weights would have a quant_state
831 - for weight_name, weight_tensor in self._hf_weight_iter(
832 - hf_weights_files, use_safetensors):
833 - # Filter out all weights whose suffix is not ".weight"
834 - if not weight_name.endswith(".weight"):
835 - continue
836 - if weight_name + ".quant_state.bitsandbytes__nf4" \
837 - in temp_state_dict:
838 - quant_state = _parse_quant_state(weight_name,
839 - temp_state_dict)
840 - weight_name = weight_name.replace(".weight", ".qweight")
841 - quant_state_dict[weight_name] = quant_state
842 - yield weight_name.replace(".weight",
843 - ".qweight"), weight_tensor
844 - else:
845 - yield weight_name, weight_tensor
846 -
847 - def generator() -> Generator:
848 - for weight_name, weight_tensor in self._hf_weight_iter(
849 - hf_weights_files, use_safetensors):
850 - if any(target_module in weight_name
851 - for target_module in self.target_modules):
852 - weight_name = weight_name.replace(".weight", ".qweight")
853 - # bitsandbytes requires data in GPU
854 - loaded_weight = weight_tensor.cuda().data
855 - with set_default_torch_dtype(torch.float32):
856 - processed_weight, quant_state = quantize_4bit(
857 - loaded_weight,
858 - compress_statistics=True,
859 - quant_type="nf4")
860 -
861 - quant_state_dict[weight_name] = quant_state
862 - else:
863 - processed_weight = weight_tensor
800 + if pre_quant:
801 + if load_8bit:
802 + return self._quantized_8bit_generator(
803 + hf_weights_files, use_safetensors,
804 + quant_state_dict), quant_state_dict
805 + else:
806 + return self._quantized_4bit_generator(
807 + hf_weights_files, use_safetensors,
808 + quant_state_dict), quant_state_dict
809
865 - yield weight_name, processed_weight
810 + return self._unquantized_generator(hf_weights_files, use_safetensors,
811 + quant_state_dict), quant_state_dict
812
867 - if pre_quant:
868 - return quantized_checkpoint(), quant_state_dict
869 - return generator(), quant_state_dict
813 + def _quantized_8bit_generator(self, hf_weights_files, use_safetensors,
814 + quant_state_dict) -> Generator:
815 + for weight_name, weight_tensor in self._hf_weight_iter(
816 + hf_weights_files, use_safetensors):
817 + if not weight_name.lower().endswith(".scb"):
818 + continue
819 +
820 + weight_key = weight_name.lower().replace(".scb", ".qweight")
821 + quant_state_dict[weight_key] = weight_tensor
822 +
823 + for weight_name, weight_tensor in self._hf_weight_iter(
824 + hf_weights_files, use_safetensors):
825 +
826 + if not weight_name.endswith(".weight"):
827 + continue
828 +
829 + qweight_name = weight_name.replace(".weight", ".qweight")
830 + if qweight_name in quant_state_dict:
831 + set_weight_attrs(weight_tensor, {"load_in_8bit": True})
832 + yield qweight_name, weight_tensor
833 + else:
834 + yield weight_name, weight_tensor
835 +
836 + def _quantized_4bit_generator(self, hf_weights_files, use_safetensors,
837 + quant_state_dict) -> Generator:
838 + from bitsandbytes.functional import QuantState
839 +
840 + # First iterate over all quant state weights
841 + weight_iterator = self._hf_weight_iter(hf_weights_files,
842 + use_safetensors)
843 + temp_state_dict = {}
844 + for weight_name, weight_tensor in weight_iterator:
845 + if weight_name.endswith(".weight"):
846 + continue
847 + # bitsandbytes library requires
848 + # weight.quant_state.bitsandbytes__* in CPU
849 + if "quant_state.bitsandbytes" in weight_name:
850 + temp_state_dict[weight_name] = weight_tensor.cpu().data
851 + else:
852 + temp_state_dict[weight_name] = weight_tensor
853 +
854 + # Closure to parse quant_state for each prequant weight
855 + def _parse_quant_state(param_name: str,
856 + temp_state_dict: Dict) -> QuantState:
857 + quant_state = {}
858 + for k in temp_state_dict:
859 + if param_name + "." in k:
860 + quant_state[k] = temp_state_dict[k]
861 +
862 + return QuantState.from_dict(quant_state, device="cuda")
863 +
864 + # Second iterate over all prequant and normal weights
865 + # pre quantized weights would have a quant_state
866 + for weight_name, weight_tensor in self._hf_weight_iter(
867 + hf_weights_files, use_safetensors):
868 + # Filter out all weights whose suffix is not ".weight"
869 + if not weight_name.endswith(".weight"):
870 + continue
871 + if (f"{weight_name}.quant_state.bitsandbytes__nf4" \
872 + in temp_state_dict) or \
873 + (f"{weight_name}.quant_state.bitsandbytes__fp4" \
874 + in temp_state_dict):
875 + quant_state = _parse_quant_state(weight_name, temp_state_dict)
876 + weight_name = weight_name.replace(".weight", ".qweight")
877 + quant_state_dict[weight_name] = quant_state
878 + yield weight_name.replace(".weight", ".qweight"), weight_tensor
879 + else:
880 + yield weight_name, weight_tensor
881 +
882 + def _unquantized_generator(self, hf_weights_files, use_safetensors,
883 + quant_state_dict) -> Generator:
884 + from bitsandbytes.functional import quantize_4bit
885 + for weight_name, weight_tensor in self._hf_weight_iter(
886 + hf_weights_files, use_safetensors):
887 + if any(target_module in weight_name
888 + for target_module in self.target_modules):
889 + weight_name = weight_name.replace(".weight", ".qweight")
890 + # bitsandbytes requires data in GPU
891 + loaded_weight = weight_tensor.cuda().data
892 + with set_default_torch_dtype(torch.float32):
893 + processed_weight, quant_state = quantize_4bit(
894 + loaded_weight,
895 + compress_statistics=True,
896 + quant_type="nf4")
897 +
898 + quant_state_dict[weight_name] = quant_state
899 + else:
900 + processed_weight = weight_tensor
901 +
902 + yield weight_name, processed_weight
903
904 def _load_weights(self, model_config: ModelConfig,
905 model: nn.Module) -> None:
@@ -883,16 +916,26 @@ class BitsAndBytesModelLoader(BaseModelLoader):
916 logger.info("Loading weights with BitsAndBytes quantization. "
917 " May take a while ...")
918
886 - is_quantized_checkpoint = False
919 quant_config = getattr(model_config.hf_config, "quantization_config",
920 None)
889 - if quant_config is not None and quant_config.get(
890 - 'quant_method') == "bitsandbytes":
891 - is_quantized_checkpoint = True
921 +
922 + pre_quant = False
923 + if quant_config is not None:
924 + quant_method = quant_config.get('quant_method')
925 + if quant_method == "bitsandbytes":
926 + pre_quant = True
927 + else:
928 + raise ValueError(
929 + f"BitsAndBytes loader does not support {quant_method} "
930 + "quantization")
931 +
932 + load_8bit = False
933 + if pre_quant:
934 + load_8bit = quant_config.get('load_in_8bit', False)
935
936 qweight_iterator, quant_state_dict = \
937 self._get_quantized_weights_iterator(
895 - model_config.model, model_config.revision, is_quantized_checkpoint)
938 + model_config.model, model_config.revision, pre_quant, load_8bit)
939
940 model.load_weights(qweight_iterator)
941
@@ -942,6 +985,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
985 offsets = np.concatenate(([0], np.cumsum(num_elements)))
986 set_weight_attrs(param, {"bnb_shard_offsets": offsets})
987
988 + if load_8bit:
989 + set_weight_attrs(
990 + param, {"matmul_state": [None] * len(quant_states)})
991 +
992 def load_model(self, *, model_config: ModelConfig,
993 device_config: DeviceConfig,
994 lora_config: Optional[LoRAConfig],