A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[bitsandbytes]: support read bnb pre-quantized model (#5753)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
dongmao zhang committed
Jul 23, 2024 at 16:45 UTC
87525fab925edf549611a1a74a40699b0b5e316e
8 files changed
+143
-39
docs/source/index.rst
+1
@@ -105,6 +105,7 @@ Documentation
105
106
quantization/supported_hardware
107
quantization/auto_awq
108
+ quantization/bnb
109
quantization/fp8
110
quantization/fp8_e5m2_kvcache
111
quantization/fp8_e4m3_kvcache
docs/source/quantization/bnb.rst
new
+43
@@ -0,0 +1,43 @@
1
+.. _bits_and_bytes:
2
+
3
+BitsAndBytes
4
+==================
5
+
6
+vLLM now supports `BitsAndBytes <https://github.com/TimDettmers/bitsandbytes>`_ for more efficient model inference.
7
+BitsAndBytes quantizes models to reduce memory usage and enhance performance without significantly sacrificing accuracy.
8
+Compared to other quantization methods, BitsAndBytes eliminates the need for calibrating the quantized model with input data.
9
+
10
+Below are the steps to utilize BitsAndBytes with vLLM.
11
+
12
+.. code-block:: console
13
+
14
+ $ pip install bitsandbytes>=0.42.0
15
+
16
+vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.
17
+
18
+You can find bitsandbytes quantized models on https://huggingface.co/models?other=bitsandbytes.
19
+And usually, these repositories have a config.json file that includes a quantization_config section.
20
+
21
+Read quantized checkpoint.
22
+--------------------------
23
+
24
+.. code-block:: python
25
+
26
+ from vllm import LLM
27
+ import torch
28
+ # unsloth/tinyllama-bnb-4bit is a pre-quantized checkpoint.
29
+ model_id = "unsloth/tinyllama-bnb-4bit"
30
+ llm = LLM(model=model_id, dtype=torch.bfloat16, trust_remote_code=True, \
31
+ quantization="bitsandbytes", load_format="bitsandbytes")
32
+
33
+Inflight quantization: load as 4bit quantization
34
+------------------------------------------------
35
+
36
+.. code-block:: python
37
+
38
+ from vllm import LLM
39
+ import torch
40
+ model_id = "huggyllama/llama-7b"
41
+ llm = LLM(model=model_id, dtype=torch.bfloat16, trust_remote_code=True, \
42
+ quantization="bitsandbytes", load_format="bitsandbytes")
43
+
tests/quantization/test_bitsandbytes.py
+14
-4
@@ -8,15 +8,20 @@ import torch
8
from tests.quantization.utils import is_quant_method_supported
9
from vllm import SamplingParams
10
11
+models_to_test = [
12
+ ('huggyllama/llama-7b', 'quantize model inflight'),
13
+ ('lllyasviel/omost-llama-3-8b-4bits', 'read pre-quantized model'),
14
+]
15
+
16
17
@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
18
reason='bitsandbytes is not supported on this GPU type.')
14
-def test_load_bnb_model(vllm_runner) -> None:
15
- with vllm_runner('huggyllama/llama-7b',
19
+@pytest.mark.parametrize("model_name, description", models_to_test)
20
+def test_load_bnb_model(vllm_runner, model_name, description) -> None:
21
+ with vllm_runner(model_name,
22
quantization='bitsandbytes',
23
load_format='bitsandbytes',
24
enforce_eager=True) as llm:
19
-
25
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
26
27
# check the weights in MLP & SelfAttention are quantized to torch.uint8
@@ -65,12 +70,17 @@ def test_load_bnb_model(vllm_runner) -> None:
70
'To be or not to be, that is the question.'
71
]
72
outputs = llm.generate(prompts, sampling_params=sampling_params)
68
-
73
assert len(outputs) == len(prompts)
74
75
for index in range(len(outputs)):
76
# compare the first line of the output
77
actual_output = outputs[index][1][0].split('\n', 1)[0]
78
expected_output = expected_outputs[index].split('\n', 1)[0]
79
+
80
+ assert len(actual_output) >= len(expected_output), (
81
+ f'Actual {actual_output} should be larger than or equal to '
82
+ f'expected {expected_output}')
83
+ actual_output = actual_output[:len(expected_output)]
84
+
85
assert actual_output == expected_output, (
86
f'Expected: {expected_output}, but got: {actual_output}')
vllm/config.py
+2
@@ -591,9 +591,11 @@ class LoadConfig:
591
mainly for profiling.
592
"tensorizer" will use CoreWeave's tensorizer library for
593
fast weight loading.
594
+ "bitsandbytes" will load nf4 type weights.
595
ignore_patterns: The list of patterns to ignore when loading the model.
596
Default to "original/**/*" to avoid repeated loading of llama's
597
checkpoints.
598
+
599
"""
600
601
load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO
vllm/engine/arg_utils.py
+2
-2
@@ -676,8 +676,8 @@ class EngineArgs:
676
# bitsandbytes quantization needs a specific model loader
677
# so we make sure the quant method and the load format are consistent
678
if (self.quantization == "bitsandbytes" or
679
- self.qlora_adapter_name_or_path is not None) and \
680
- self.load_format != "bitsandbytes":
679
+ self.qlora_adapter_name_or_path is not None) and \
680
+ self.load_format != "bitsandbytes":
681
raise ValueError(
682
"BitsAndBytes quantization and QLoRA adapter only support "
683
f"'bitsandbytes' load format, but got {self.load_format}")
vllm/model_executor/layers/quantization/bitsandbytes.py
+4
-21
@@ -15,19 +15,11 @@ class BitsAndBytesConfig(QuantizationConfig):
15
Reference: https://arxiv.org/abs/2305.14314
16
"""
17
18
- def __init__(
19
- self,
20
- adapter_name_or_path: str,
21
- target_modules: List[str],
22
- ) -> None:
23
-
24
- self.adapter_name_or_path = adapter_name_or_path
25
- self.target_modules = target_modules
18
+ def __init__(self, ) -> None:
19
+ pass
20
21
def __repr__(self) -> str:
28
- return (
29
- f"BitsAndBytesConfig(adapter_name_or_path={self.adapter_name_or_path}"
30
- )
22
+ return "BitsAndBytesConfig"
23
24
@classmethod
25
def get_name(self) -> str:
@@ -49,16 +41,7 @@ class BitsAndBytesConfig(QuantizationConfig):
41
42
@classmethod
43
def from_config(cls, config: Dict[str, Any]) -> "BitsAndBytesConfig":
52
- adapter_name = cls.get_from_keys(config, ["adapter_name_or_path"])
53
- default_target_modules = [
54
- "gate_proj", "down_proj", "up_proj", "q_proj", "k_proj", "v_proj",
55
- "o_proj"
56
- ]
57
- if adapter_name == "":
58
- target_modules = default_target_modules
59
- else:
60
- target_modules = cls.get_from_keys(config, ["target_modules"])
61
- return cls(adapter_name, target_modules)
44
+ return cls()
45
46
def get_quant_method(self, layer: torch.nn.Module,
47
prefix: str) -> Optional["BitsAndBytesLinearMethod"]:
vllm/model_executor/model_loader/loader.py
+76
-12
@@ -702,8 +702,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
702
703
return hf_weights_files, matched_pattern == "*.safetensors"
704
705
+ def _hf_weight_iter(self, hf_weights_files, use_safetensors: bool):
706
+ if use_safetensors:
707
+ return safetensors_weights_iterator(hf_weights_files)
708
+ else:
709
+ return pt_weights_iterator(hf_weights_files)
710
+
711
def _get_quantized_weights_iterator(
706
- self, model_name_or_path: str, revision: Optional[str]
712
+ self, model_name_or_path: str, revision: Optional[str], pre_quant: bool
713
) -> Tuple[Generator[Tuple[str, torch.Tensor], None, None], Dict[str,
714
Any]]:
715
"""Get an iterator to the model weights with bitsandbytes quantization,
@@ -712,6 +718,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
718
# only load the bitsandbytes module when needed
719
try:
720
import bitsandbytes
721
+ from bitsandbytes.functional import QuantState
722
if bitsandbytes.__version__ < "0.42.0":
723
raise ImportError("bitsandbytes version is wrong. Please "
724
"install bitsandbytes>=0.42.0.")
@@ -725,17 +732,63 @@ class BitsAndBytesModelLoader(BaseModelLoader):
732
model_name_or_path, revision)
733
734
quant_state_dict = {}
728
- if use_safetensors:
729
- weight_iterator = safetensors_weights_iterator(hf_weights_files)
730
- else:
731
- weight_iterator = pt_weights_iterator(hf_weights_files)
735
733
- def generator():
736
+ def quantized_checkpoint() -> Generator:
737
+ # First iterate over all quant state weights
738
+ weight_iterator = self._hf_weight_iter(hf_weights_files,
739
+ use_safetensors)
740
+ temp_state_dict = {}
741
for weight_name, weight_tensor in weight_iterator:
742
+ if weight_name.endswith(".weight"):
743
+ continue
744
+ # TODO: only nf4 quantization is supported for now
745
+ if weight_name.endswith(".quant_state.bitsandbytes__fp4"):
746
+ raise NotImplementedError(
747
+ "Only bitsandbytes_nf4 quantization"
748
+ f"is supported for now. {weight_name} is fp4 quantized"
749
+ )
750
+ temp_state_dict[weight_name] = weight_tensor
751
+
752
+ # Closure to parse quant_state for each prequant weight
753
+ def _parse_quant_state(param_name: str,
754
+ temp_state_dict: Dict) -> QuantState:
755
+ quant_state = {}
756
+ for k in temp_state_dict:
757
+ if param_name + "." in k:
758
+ quant_state[k] = temp_state_dict[k]
759
+ # bitsandbytes library requires
760
+ # weight.quant_state.bitsandbytes__nf4 in CPU
761
+ quant_state[param_name +
762
+ ".quant_state.bitsandbytes__nf4"] = quant_state[
763
+ param_name +
764
+ ".quant_state.bitsandbytes__nf4"].cpu().data
765
+ return QuantState.from_dict(quant_state, device="cuda")
766
+
767
+ # Second iterate over all prequant and normal weights
768
+ # pre quantized weights would have a quant_state
769
+ for weight_name, weight_tensor in self._hf_weight_iter(
770
+ hf_weights_files, use_safetensors):
771
+ # Filter out all weights whose suffix is not ".weight"
772
+ if not weight_name.endswith(".weight"):
773
+ continue
774
+ if weight_name + ".quant_state.bitsandbytes__nf4" \
775
+ in temp_state_dict:
776
+ quant_state = _parse_quant_state(weight_name,
777
+ temp_state_dict)
778
+ weight_name = weight_name.replace(".weight", ".qweight")
779
+ quant_state_dict[weight_name] = quant_state
780
+ yield weight_name.replace(".weight",
781
+ ".qweight"), weight_tensor
782
+ else:
783
+ yield weight_name, weight_tensor
784
+
785
+ def generator() -> Generator:
786
+ for weight_name, weight_tensor in self._hf_weight_iter(
787
+ hf_weights_files, use_safetensors):
788
if any(target_module in weight_name
789
for target_module in self.target_modules):
790
weight_name = weight_name.replace(".weight", ".qweight")
738
- # bitsandbytes requires data in GPU
791
+ # bitsandbytes requires data in GPU
792
loaded_weight = weight_tensor.cuda().data
793
with set_default_torch_dtype(torch.float32):
794
processed_weight, quant_state = quantize_4bit(
@@ -749,6 +802,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
802
803
yield weight_name, processed_weight
804
805
+ if pre_quant:
806
+ return quantized_checkpoint(), quant_state_dict
807
return generator(), quant_state_dict
808
809
def _load_weights(self, model_config: ModelConfig,
@@ -766,12 +821,21 @@ class BitsAndBytesModelLoader(BaseModelLoader):
821
logger.info("Loading weights with BitsAndBytes quantization. "
822
" May take a while ...")
823
769
- qweight_iterator, quant_state_dict = (
770
- self._get_quantized_weights_iterator(model_config.model,
771
- model_config.revision))
824
+ is_quantized_checkpoint = False
825
+ quant_config = getattr(model_config.hf_config, "quantization_config",
826
+ None)
827
+ if quant_config is not None and quant_config.get(
828
+ 'quant_method') == "bitsandbytes":
829
+ is_quantized_checkpoint = True
830
+
831
+ qweight_iterator, quant_state_dict = \
832
+ self._get_quantized_weights_iterator(
833
+ model_config.model, model_config.revision, is_quantized_checkpoint)
834
835
model.load_weights(qweight_iterator)
836
837
+ torch.cuda.empty_cache()
838
+
839
param_dict = dict(model.named_parameters())
840
stacked_quant_state_dict: Dict[str, Dict[int, Any]] = {}
841
for quant_param_name in quant_state_dict:
@@ -809,9 +873,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
873
f"pack_factor not set for parameter {param_name}.")
874
875
num_elements = [0] * len(quant_states)
812
- for seq, quant_state in enumerate(quant_states.items()):
876
+ for seq, quant_state in quant_states.items():
877
num_elements[seq] = math.prod(
814
- quant_state[1].shape) // pack_ratio
878
+ quant_state.shape) // pack_ratio
879
880
offsets = np.concatenate(([0], np.cumsum(num_elements)))
881
set_weight_attrs(param, {"bnb_shard_offsets": offsets})
vllm/model_executor/model_loader/weight_utils.py
+1
@@ -118,6 +118,7 @@ def convert_bin_to_safetensor_file(
118
# TODO(woosuk): Move this to other place.
119
def get_quant_config(model_config: ModelConfig,
120
load_config: LoadConfig) -> QuantizationConfig:
121
+
122
quant_cls = get_quantization_config(model_config.quantization)
123
# Read the quantization config from the HF model config, if available.
124
hf_quant_config = getattr(model_config.hf_config, "quantization_config",