A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Feature][Kernel] Support bitsandbytes quantization and QLoRA (#4776)
chenqianfzh committed
Jun 1, 2024 at 13:51 UTC
b9c0605a8e7d558f595bd59ba6e6c95578dc0f1e
11 files changed
+752
-8
examples/lora_with_quantization_inference.py
new
+140
@@ -0,0 +1,140 @@
1
+"""
2
+This example shows how to use LoRA with different quantization techniques
3
+for offline inference.
4
+
5
+Requires HuggingFace credentials for access.
6
+"""
7
+
8
+import gc
9
+from typing import List, Optional, Tuple
10
+
11
+import torch
12
+from huggingface_hub import snapshot_download
13
+
14
+from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
15
+from vllm.lora.request import LoRARequest
16
+
17
+
18
+def create_test_prompts(
19
+ lora_path: str
20
+) -> List[Tuple[str, SamplingParams, Optional[LoRARequest]]]:
21
+ return [
22
+ # this is an example of using quantization without LoRA
23
+ ("My name is",
24
+ SamplingParams(temperature=0.0,
25
+ logprobs=1,
26
+ prompt_logprobs=1,
27
+ max_tokens=128), None),
28
+ # the next three examples use quantization with LoRA
29
+ ("my name is",
30
+ SamplingParams(temperature=0.0,
31
+ logprobs=1,
32
+ prompt_logprobs=1,
33
+ max_tokens=128),
34
+ LoRARequest("lora-test-1", 1, lora_path)),
35
+ ("The capital of USA is",
36
+ SamplingParams(temperature=0.0,
37
+ logprobs=1,
38
+ prompt_logprobs=1,
39
+ max_tokens=128),
40
+ LoRARequest("lora-test-2", 1, lora_path)),
41
+ ("The capital of France is",
42
+ SamplingParams(temperature=0.0,
43
+ logprobs=1,
44
+ prompt_logprobs=1,
45
+ max_tokens=128),
46
+ LoRARequest("lora-test-3", 1, lora_path)),
47
+ ]
48
+
49
+
50
+def process_requests(engine: LLMEngine,
51
+ test_prompts: List[Tuple[str, SamplingParams,
52
+ Optional[LoRARequest]]]):
53
+ """Continuously process a list of prompts and handle the outputs."""
54
+ request_id = 0
55
+
56
+ while test_prompts or engine.has_unfinished_requests():
57
+ if test_prompts:
58
+ prompt, sampling_params, lora_request = test_prompts.pop(0)
59
+ engine.add_request(str(request_id),
60
+ prompt,
61
+ sampling_params,
62
+ lora_request=lora_request)
63
+ request_id += 1
64
+
65
+ request_outputs: List[RequestOutput] = engine.step()
66
+ for request_output in request_outputs:
67
+ if request_output.finished:
68
+ print("----------------------------------------------------")
69
+ print(f"Prompt: {request_output.prompt}")
70
+ print(f"Output: {request_output.outputs[0].text}")
71
+
72
+
73
+def initialize_engine(model: str, quantization: str,
74
+ lora_repo: Optional[str]) -> LLMEngine:
75
+ """Initialize the LLMEngine."""
76
+
77
+ if quantization == "bitsandbytes":
78
+ # QLoRA (https://arxiv.org/abs/2305.14314) is a quantization technique.
79
+ # It quantizes the model when loading, with some config info from the
80
+ # LoRA adapter repo. So need to set the parameter of load_format and
81
+ # qlora_adapter_name_or_path as below.
82
+ engine_args = EngineArgs(
83
+ model=model,
84
+ quantization=quantization,
85
+ qlora_adapter_name_or_path=lora_repo,
86
+ load_format="bitsandbytes",
87
+ enable_lora=True,
88
+ max_lora_rank=64,
89
+ # set it only in GPUs of limited memory
90
+ enforce_eager=True)
91
+ else:
92
+ engine_args = EngineArgs(
93
+ model=model,
94
+ quantization=quantization,
95
+ enable_lora=True,
96
+ max_loras=4,
97
+ # set it only in GPUs of limited memory
98
+ enforce_eager=True)
99
+ return LLMEngine.from_engine_args(engine_args)
100
+
101
+
102
+def main():
103
+ """Main function that sets up and runs the prompt processing."""
104
+
105
+ test_configs = [{
106
+ "name": "qlora_inference_example",
107
+ 'model': "huggyllama/llama-7b",
108
+ 'quantization': "bitsandbytes",
109
+ 'lora_repo': 'timdettmers/qlora-flan-7b'
110
+ }, {
111
+ "name": "AWQ_inference_with_lora_example",
112
+ 'model': 'TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ',
113
+ 'quantization': "awq",
114
+ 'lora_repo': 'jashing/tinyllama-colorist-lora'
115
+ }, {
116
+ "name": "GPTQ_inference_with_lora_example",
117
+ 'model': 'TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ',
118
+ 'quantization': "gptq",
119
+ 'lora_repo': 'jashing/tinyllama-colorist-lora'
120
+ }]
121
+
122
+ for test_config in test_configs:
123
+ print(
124
+ f"~~~~~~~~~~~~~~~~ Running: {test_config['name']} ~~~~~~~~~~~~~~~~"
125
+ )
126
+ engine = initialize_engine(test_config['model'],
127
+ test_config['quantization'],
128
+ test_config['lora_repo'])
129
+ lora_path = snapshot_download(repo_id=test_config['lora_repo'])
130
+ test_prompts = create_test_prompts(lora_path)
131
+ process_requests(engine, test_prompts)
132
+
133
+ # Clean up the GPU memory for the next test
134
+ del engine
135
+ gc.collect()
136
+ torch.cuda.empty_cache()
137
+
138
+
139
+if __name__ == '__main__':
140
+ main()
requirements-dev.txt
+3
@@ -35,3 +35,6 @@ aiohttp
35
36
# Multimodal
37
pillow
38
+
39
+# quantization
40
+bitsandbytes==0.42.0
tests/quantization/test_bitsandbytes.py
new
+80
@@ -0,0 +1,80 @@
1
+'''Tests whether bitsandbytes computation is enabled correctly.
2
+
3
+Run `pytest tests/quantization/test_bitsandbytes.py`.
4
+'''
5
+import pytest
6
+import torch
7
+
8
+from vllm import SamplingParams
9
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
10
+
11
+capability = torch.cuda.get_device_capability()
12
+capability = capability[0] * 10 + capability[1]
13
+
14
+
15
+@pytest.mark.skipif(
16
+ capability < QUANTIZATION_METHODS['bitsandbytes'].get_min_capability(),
17
+ reason='bitsandbytes is not supported on this GPU type.')
18
+def test_load_bnb_model(vllm_runner) -> None:
19
+ llm = vllm_runner('huggyllama/llama-7b',
20
+ quantization='bitsandbytes',
21
+ load_format='bitsandbytes',
22
+ enforce_eager=True)
23
+
24
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
25
+
26
+ # check the weights in MLP & SelfAttention are quantized to torch.uint8
27
+ qweight = model.model.layers[0].mlp.gate_up_proj.qweight
28
+ assert qweight.dtype == torch.uint8, (
29
+ f'Expected gate_up_proj dtype torch.uint8 but got {qweight.dtype}')
30
+
31
+ qweight = model.model.layers[0].mlp.down_proj.qweight
32
+ assert qweight.dtype == torch.uint8, (
33
+ f'Expected down_proj dtype torch.uint8 but got {qweight.dtype}')
34
+
35
+ qweight = model.model.layers[0].self_attn.o_proj.qweight
36
+ assert qweight.dtype == torch.uint8, (
37
+ f'Expected o_proj dtype torch.uint8 but got {qweight.dtype}')
38
+
39
+ qweight = model.model.layers[0].self_attn.qkv_proj.qweight
40
+ assert qweight.dtype == torch.uint8, (
41
+ f'Expected qkv_proj dtype torch.uint8 but got {qweight.dtype}')
42
+
43
+ # some weights should not be quantized
44
+ weight = model.lm_head.weight
45
+ assert weight.dtype != torch.uint8, (
46
+ 'lm_head weight dtype should not be torch.uint8')
47
+
48
+ weight = model.model.embed_tokens.weight
49
+ assert weight.dtype != torch.uint8, (
50
+ 'embed_tokens weight dtype should not be torch.uint8')
51
+
52
+ weight = model.model.layers[0].input_layernorm.weight
53
+ assert weight.dtype != torch.uint8, (
54
+ 'input_layernorm weight dtype should not be torch.uint8')
55
+
56
+ weight = model.model.layers[0].post_attention_layernorm.weight
57
+ assert weight.dtype != torch.uint8, (
58
+ 'input_layernorm weight dtype should not be torch.uint8')
59
+
60
+ # check the output of the model is expected
61
+ sampling_params = SamplingParams(temperature=0.0,
62
+ logprobs=1,
63
+ prompt_logprobs=1,
64
+ max_tokens=8)
65
+
66
+ prompts = ['That which does not kill us', 'To be or not to be,']
67
+ expected_outputs = [
68
+ 'That which does not kill us makes us stronger.',
69
+ 'To be or not to be, that is the question.'
70
+ ]
71
+ outputs = llm.generate(prompts, sampling_params=sampling_params)
72
+
73
+ assert len(outputs) == len(prompts)
74
+
75
+ for index in range(len(outputs)):
76
+ # compare the first line of the output
77
+ actual_output = outputs[index][1][0].split('\n', 1)[0]
78
+ expected_output = expected_outputs[index].split('\n', 1)[0]
79
+ assert actual_output == expected_output, (
80
+ f'Expected: {expected_output}, but got: {actual_output}')
vllm/config.py
+8
-1
@@ -241,6 +241,12 @@ class ModelConfig:
241
"must be divisible by pipeline parallel size "
242
f"({pipeline_parallel_size}).")
243
244
+ if self.quantization == "bitsandbytes" and (
245
+ parallel_config.tensor_parallel_size > 1
246
+ or parallel_config.pipeline_parallel_size > 1):
247
+ raise ValueError(
248
+ "BitAndBytes quantization with TP or PP is not supported yet.")
249
+
250
def get_hf_config_sliding_window(self) -> Optional[int]:
251
"""Get the sliding window size, or None if disabled.
252
"""
@@ -327,7 +333,7 @@ class ModelConfig:
333
def get_num_attention_heads(self,
334
parallel_config: "ParallelConfig") -> int:
335
return self.hf_text_config.num_attention_heads // \
330
- parallel_config.tensor_parallel_size
336
+ parallel_config.tensor_parallel_size
337
338
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
339
total_num_hidden_layers = self.hf_text_config.num_hidden_layers
@@ -487,6 +493,7 @@ class LoadFormat(str, enum.Enum):
493
DUMMY = "dummy"
494
TENSORIZER = "tensorizer"
495
SHARDED_STATE = "sharded_state"
496
+ BITSANDBYTES = "bitsandbytes"
497
498
499
@dataclass
vllm/engine/arg_utils.py
+35
-3
@@ -92,6 +92,8 @@ class EngineArgs:
92
ngram_prompt_lookup_max: Optional[int] = None
93
ngram_prompt_lookup_min: Optional[int] = None
94
95
+ qlora_adapter_name_or_path: Optional[str] = None
96
+
97
def __post_init__(self):
98
if self.tokenizer is None:
99
self.tokenizer = self.model
@@ -159,7 +161,8 @@ class EngineArgs:
161
type=str,
162
default=EngineArgs.load_format,
163
choices=[
162
- 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer'
164
+ 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer',
165
+ 'bitsandbytes'
166
],
167
help='The format of the model weights to load.\n\n'
168
'* "auto" will try to load the weights in the safetensors format '
@@ -173,7 +176,9 @@ class EngineArgs:
176
'which is mainly for profiling.\n'
177
'* "tensorizer" will load the weights using tensorizer from '
178
'CoreWeave. See the Tensorize vLLM Model script in the Examples'
176
- 'section for more information.\n')
179
+ 'section for more information.\n'
180
+ '* "bitsandbytes" will load the weights using bitsandbytes '
181
+ 'quantization.\n')
182
parser.add_argument(
183
'--dtype',
184
type=str,
@@ -543,7 +548,10 @@ class EngineArgs:
548
"will also be used in `model_name` tag content of "
549
"prometheus metrics, if multiple names provided, metrics"
550
"tag will take the first one.")
546
-
551
+ parser.add_argument('--qlora-adapter-name-or-path',
552
+ type=str,
553
+ default=None,
554
+ help='Name or path of the QLoRA adapter.')
555
return parser
556
557
@classmethod
@@ -555,6 +563,23 @@ class EngineArgs:
563
return engine_args
564
565
def create_engine_config(self, ) -> EngineConfig:
566
+
567
+ # bitsandbytes quantization needs a specific model loader
568
+ # so we make sure the quant method and the load format are consistent
569
+ if (self.quantization == "bitsandbytes" or
570
+ self.qlora_adapter_name_or_path is not None) and \
571
+ self.load_format != "bitsandbytes":
572
+ raise ValueError(
573
+ "BitsAndBytes quantization and QLoRA adapter only support "
574
+ f"'bitsandbytes' load format, but got {self.load_format}")
575
+
576
+ if (self.load_format == "bitsandbytes" or
577
+ self.qlora_adapter_name_or_path is not None) and \
578
+ self.quantization != "bitsandbytes":
579
+ raise ValueError(
580
+ "BitsAndBytes load format and QLoRA adapter only support "
581
+ f"'bitsandbytes' quantization, but got {self.quantization}")
582
+
583
device_config = DeviceConfig(self.device)
584
model_config = ModelConfig(
585
self.model, self.tokenizer, self.tokenizer_mode,
@@ -622,6 +647,13 @@ class EngineArgs:
647
max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
648
and self.max_cpu_loras > 0 else None) if self.enable_lora else None
649
650
+ if self.qlora_adapter_name_or_path is not None and \
651
+ self.qlora_adapter_name_or_path != "":
652
+ if self.model_loader_extra_config is None:
653
+ self.model_loader_extra_config = {}
654
+ self.model_loader_extra_config[
655
+ "qlora_adapter_name_or_path"] = self.qlora_adapter_name_or_path
656
+
657
load_config = LoadConfig(
658
load_format=self.load_format,
659
download_dir=self.download_dir,
vllm/model_executor/layers/linear.py
+39
-2
@@ -1,5 +1,5 @@
1
from abc import abstractmethod
2
-from typing import List, Optional
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
import torch.nn.functional as F
@@ -26,6 +26,21 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
26
return shard_size * marlin_tile_size, shard_offset * marlin_tile_size
27
28
29
+def adjust_bitsandbytes_shard(param: Parameter,
30
+ qkv_offsets: Dict[str, Tuple[int, int]],
31
+ loaded_shard_id: str) -> Tuple[int, int]:
32
+ """Adjust the quantization offsets and sizes for BitsAndBytes sharding."""
33
+
34
+ total, _ = qkv_offsets["total"]
35
+ orig_offset, orig_size = qkv_offsets[loaded_shard_id]
36
+
37
+ quantized_total = param.data.shape[0]
38
+ quantized_offset = orig_offset * quantized_total // total
39
+ quantized_size = orig_size * quantized_total // total
40
+
41
+ return quantized_size, quantized_offset
42
+
43
+
44
class LinearMethodBase(QuantizeMethodBase):
45
"""Base class for different (maybe quantized) linear methods."""
46
@@ -37,7 +52,7 @@ class LinearMethodBase(QuantizeMethodBase):
52
**extra_weight_attrs):
53
"""Create weights for a linear layer.
54
The weights will be set as attributes of the layer.
40
-
55
+
56
Args:
57
layer: The layer that is using the LinearMethodBase factory.
58
input_size_per_partition: Size of the weight input dim on rank X.
@@ -416,6 +431,12 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
431
shard_size, shard_offset = adjust_marlin_shard(
432
param, shard_size, shard_offset)
433
434
+ use_bitsandbytes = getattr(param, "use_bitsandbytes", False)
435
+ if use_bitsandbytes:
436
+ shard_size = loaded_weight.shape[output_dim]
437
+ shard_offset = loaded_weight.shape[output_dim] * \
438
+ loaded_shard_id
439
+
440
param_data = param_data.narrow(output_dim, shard_offset,
441
shard_size)
442
start_idx = tp_rank * shard_size
@@ -615,6 +636,22 @@ class QKVParallelLinear(ColumnParallelLinear):
636
shard_size, shard_offset = adjust_marlin_shard(
637
param, shard_size, shard_offset)
638
639
+ use_bitsandbytes = getattr(param, "use_bitsandbytes", False)
640
+ if use_bitsandbytes:
641
+ orig_qkv_offsets = {
642
+ "q": (0, self.num_heads * self.head_size),
643
+ "k": (self.num_heads * self.head_size,
644
+ self.num_kv_heads * self.head_size),
645
+ "v":
646
+ ((self.num_heads + self.num_kv_heads) * self.head_size,
647
+ self.num_kv_heads * self.head_size),
648
+ "total":
649
+ ((self.num_heads + 2 * self.num_kv_heads) * self.head_size,
650
+ 0)
651
+ }
652
+ shard_size, shard_offset = adjust_bitsandbytes_shard(
653
+ param, orig_qkv_offsets, loaded_shard_id)
654
+
655
param_data = param_data.narrow(output_dim, shard_offset,
656
shard_size)
657
if loaded_shard_id == "q":
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -4,6 +4,8 @@ from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4
from vllm.model_executor.layers.quantization.awq import AWQConfig
5
from vllm.model_executor.layers.quantization.base_config import (
6
QuantizationConfig)
7
+from vllm.model_executor.layers.quantization.bitsandbytes import (
8
+ BitsAndBytesConfig)
9
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10
CompressedTensorsConfig)
11
from vllm.model_executor.layers.quantization.deepspeedfp import (
@@ -30,6 +32,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
32
"gptq": GPTQConfig,
33
"squeezellm": SqueezeLLMConfig,
34
"sparseml": CompressedTensorsConfig,
35
+ "bitsandbytes": BitsAndBytesConfig,
36
}
37
38
vllm/model_executor/layers/quantization/bitsandbytes.py
new
+175
@@ -0,0 +1,175 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+from torch.nn.parameter import Parameter
5
+
6
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
7
+ set_weight_attrs)
8
+from vllm.model_executor.layers.quantization.base_config import (
9
+ QuantizationConfig)
10
+
11
+
12
+class BitsAndBytesConfig(QuantizationConfig):
13
+ """Config class for BitsAndBytes Quantization.
14
+
15
+ Reference: https://arxiv.org/abs/2305.14314
16
+ """
17
+
18
+ def __init__(
19
+ self,
20
+ adapter_name_or_path: str,
21
+ target_modules: List[str],
22
+ ) -> None:
23
+
24
+ self.adapter_name_or_path = adapter_name_or_path
25
+ self.target_modules = target_modules
26
+
27
+ def __repr__(self) -> str:
28
+ return (
29
+ f"BitsAndBytesConfig(adapter_name_or_path={self.adapter_name_or_path}"
30
+ )
31
+
32
+ @classmethod
33
+ def get_name(self) -> str:
34
+ return "bitsandbytes"
35
+
36
+ @classmethod
37
+ def get_supported_act_dtypes(self) -> List[torch.dtype]:
38
+ return [torch.float32, torch.float16, torch.bfloat16]
39
+
40
+ @classmethod
41
+ def get_min_capability(self) -> int:
42
+ return 70
43
+
44
+ @staticmethod
45
+ def get_config_filenames() -> List[str]:
46
+ return [
47
+ "adapter_config.json",
48
+ ]
49
+
50
+ @classmethod
51
+ def from_config(cls, config: Dict[str, Any]) -> "BitsAndBytesConfig":
52
+ adapter_name = cls.get_from_keys(config, ["adapter_name_or_path"])
53
+ default_target_modules = [
54
+ "gate_proj", "down_proj", "up_proj", "q_proj", "k_proj", "v_proj",
55
+ "o_proj"
56
+ ]
57
+ if adapter_name == "":
58
+ target_modules = default_target_modules
59
+ else:
60
+ target_modules = cls.get_from_keys(config, ["target_modules"])
61
+ return cls(adapter_name, target_modules)
62
+
63
+ def get_quant_method(
64
+ self,
65
+ layer: torch.nn.Module) -> Optional["BitsAndBytesLinearMethod"]:
66
+ if isinstance(layer, LinearBase):
67
+ return BitsAndBytesLinearMethod(self)
68
+ return None
69
+
70
+ def get_scaled_act_names(self) -> List[str]:
71
+ return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
72
+
73
+
74
+class BitsAndBytesLinearMethod(LinearMethodBase):
75
+ """Linear method for BitsAndBytes.
76
+
77
+ Args:
78
+ quant_config: The BitsAndBytes quantization config.
79
+ """
80
+
81
+ def __init__(self, quant_config: BitsAndBytesConfig):
82
+ try:
83
+ import bitsandbytes
84
+ if bitsandbytes.__version__ < "0.42.0":
85
+ raise ImportError("bitsandbytes version is wrong. Please "
86
+ "install bitsandbytes>=0.42.0.")
87
+ except ImportError as err:
88
+ raise ImportError("Please install bitsandbytes>=0.42.0 via "
89
+ "`pip install bitsandbytes>=0.42.0` to use "
90
+ "bitsandbytes quantizer.") from err
91
+
92
+ self.quant_config = quant_config
93
+
94
+ def create_weights(self, layer: torch.nn.Module,
95
+ input_size_per_partition: int,
96
+ output_partition_sizes: List[int], input_size: int,
97
+ output_size: int, params_dtype: torch.dtype,
98
+ **extra_weight_attrs):
99
+ quant_ratio = 0
100
+ if params_dtype.is_floating_point:
101
+ quant_ratio = torch.finfo(params_dtype).bits // torch.iinfo(
102
+ torch.uint8).bits
103
+ else:
104
+ quant_ratio = torch.iinfo(params_dtype).bits // torch.iinfo(
105
+ torch.uint8).bits
106
+
107
+ if input_size_per_partition * sum(
108
+ output_partition_sizes) % quant_ratio != 0:
109
+ raise ValueError(
110
+ "The input size is not aligned with the quantized "
111
+ "weight shape. ")
112
+ qweight = Parameter(
113
+ torch.empty(
114
+ input_size_per_partition * sum(output_partition_sizes) //
115
+ quant_ratio,
116
+ 1,
117
+ dtype=torch.uint8,
118
+ ),
119
+ requires_grad=False,
120
+ )
121
+
122
+ set_weight_attrs(
123
+ qweight,
124
+ {
125
+ "input_dim": 0,
126
+ # In bitsandbytes, a tensor of shape [n,m] is quantized to
127
+ #[n*m/pack_ratio, 1],so the output_dim is 0
128
+ "output_dim": 0,
129
+ "pack_factor": quant_ratio,
130
+ "use_bitsandbytes": True,
131
+ })
132
+ layer.register_parameter("qweight", qweight)
133
+ set_weight_attrs(qweight, extra_weight_attrs)
134
+
135
+ def apply(self,
136
+ layer: torch.nn.Module,
137
+ x: torch.Tensor,
138
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
139
+
140
+ # only load the bitsandbytes module when needed
141
+ from bitsandbytes import matmul_4bit
142
+
143
+ original_type = x.dtype
144
+ bf_x = x.to(torch.bfloat16)
145
+
146
+ qweight = layer.qweight
147
+ quant_states = qweight.bnb_quant_state
148
+ offsets = qweight.bnb_shard_offsets
149
+
150
+ out_dim_0 = x.shape[0]
151
+ out_dim_1 = sum(
152
+ [quant_state[1].shape[0] for quant_state in quant_states.items()])
153
+ out = torch.empty(out_dim_0,
154
+ out_dim_1,
155
+ dtype=torch.bfloat16,
156
+ device=x.device)
157
+
158
+ current_index = 0
159
+ for i in range(len(quant_states)):
160
+ output_size = quant_states[i].shape[0]
161
+ # It is more efficient to use out kwarg like
162
+ # matmul_4bit(..., out = ...). Infeasible now due to the bug
163
+ # https://github.com/TimDettmers/bitsandbytes/issues/1235.
164
+ # Need to change after the bug is fixed.
165
+ out[:, current_index:current_index + output_size] = matmul_4bit(
166
+ bf_x, qweight[offsets[i]:offsets[i + 1]].t(), quant_states[i])
167
+
168
+ current_index += output_size
169
+
170
+ out = out.to(original_type)
171
+
172
+ if bias is not None:
173
+ out += bias
174
+
175
+ return out
vllm/model_executor/model_loader/loader.py
+246
-1
@@ -1,13 +1,18 @@
1
# ruff: noqa: SIM117
2
import collections
3
import copy
4
+import fnmatch
5
import glob
6
+import json
7
+import math
8
import os
9
from abc import ABC, abstractmethod
10
from typing import Any, Dict, Generator, List, Optional, Tuple, Type
11
12
import huggingface_hub
13
+import numpy as np
14
import torch
15
+from huggingface_hub import HfApi, hf_hub_download
16
from torch import nn
17
18
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
@@ -28,6 +33,7 @@ from vllm.model_executor.model_loader.weight_utils import (
33
get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
34
pt_weights_iterator, safetensors_weights_iterator)
35
from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
36
+from vllm.model_executor.utils import set_weight_attrs
37
38
logger = init_logger(__name__)
39
@@ -125,7 +131,7 @@ class DefaultModelLoader(BaseModelLoader):
131
def _maybe_download_from_modelscope(
132
self, model: str, revision: Optional[str]) -> Optional[str]:
133
"""Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True.
128
-
134
+
135
Returns the path to the downloaded model, or None if the model is not
136
downloaded from ModelScope."""
137
if VLLM_USE_MODELSCOPE:
@@ -247,6 +253,7 @@ class DefaultModelLoader(BaseModelLoader):
253
model,
254
"fall_back_to_pt_during_load",
255
True)), )
256
+
257
for _, module in model.named_modules():
258
quant_method = getattr(module, "quant_method", None)
259
if quant_method is not None:
@@ -539,6 +546,241 @@ class ShardedStateLoader(BaseModelLoader):
546
)
547
548
549
+class BitsAndBytesModelLoader(BaseModelLoader):
550
+ """Model loader to load model weights with BitAndBytes quantization."""
551
+
552
+ default_target_modules = [
553
+ "gate_proj", "down_proj", "up_proj", "q_proj", "k_proj", "v_proj",
554
+ "o_proj"
555
+ ]
556
+
557
+ possible_config_file_names = ["adapter_config.json"]
558
+
559
+ def __init__(self, load_config: LoadConfig):
560
+ super().__init__(load_config)
561
+
562
+ # we don't need to quantize the whole model, only the target modules
563
+ # that are specified in the adapter config file. If the adapter config
564
+ # file is not provided, we will quantize the default modules.
565
+ if (not load_config.model_loader_extra_config
566
+ or "qlora_adapter_name_or_path"
567
+ not in load_config.model_loader_extra_config):
568
+ self.target_modules = self.default_target_modules
569
+ return
570
+
571
+ qlora_adapter = load_config.model_loader_extra_config[
572
+ "qlora_adapter_name_or_path"]
573
+
574
+ config_file_path = self._get_config_file(qlora_adapter)
575
+
576
+ with open(config_file_path, "r") as f:
577
+ config = json.load(f)
578
+ self.target_modules = config["target_modules"]
579
+
580
+ def _get_config_file(self, qlora_adapter: str) -> str:
581
+ is_local = os.path.isdir(qlora_adapter)
582
+ config_file_path = None
583
+ if is_local:
584
+ for file in self.possible_config_file_names:
585
+ config_file_path = os.path.join(qlora_adapter, file)
586
+ if os.path.exists(config_file_path):
587
+ break
588
+ else:
589
+ hf_api = HfApi()
590
+ repo_files = hf_api.list_repo_files(repo_id=qlora_adapter)
591
+ for file in self.possible_config_file_names:
592
+ if file in repo_files:
593
+ config_file_path = hf_hub_download(repo_id=qlora_adapter,
594
+ filename=file)
595
+ break
596
+
597
+ if not config_file_path:
598
+ raise ValueError(
599
+ f"Cannot find adapter config file in {qlora_adapter}")
600
+
601
+ return config_file_path
602
+
603
+ def _get_weight_files(
604
+ self,
605
+ model_name_or_path: str,
606
+ allowed_patterns: List[str],
607
+ revision: Optional[str] = None) -> Tuple[List[str], str]:
608
+ """Retrieve weight files. Download the files if necessary.
609
+
610
+ Return the weight files and the file pattern."""
611
+ is_local = os.path.isdir(model_name_or_path)
612
+
613
+ if is_local:
614
+ for pattern in allowed_patterns:
615
+ weight_files = glob.glob(
616
+ os.path.join(model_name_or_path, pattern))
617
+ if weight_files:
618
+ return weight_files, pattern
619
+ else:
620
+ hf_api = HfApi()
621
+ repo_files = hf_api.list_repo_files(repo_id=model_name_or_path)
622
+ for pattern in allowed_patterns:
623
+ matching_files = fnmatch.filter(repo_files, pattern)
624
+ if matching_files:
625
+ hf_folder = download_weights_from_hf(
626
+ model_name_or_path, self.load_config.download_dir,
627
+ [pattern], revision)
628
+ return glob.glob(os.path.join(hf_folder, pattern)), pattern
629
+
630
+ raise RuntimeError(
631
+ f"No model weights found in: `{model_name_or_path}`")
632
+
633
+ def _prepare_weights(self, model_name_or_path: str,
634
+ revision: Optional[str]) -> Tuple[List[str], bool]:
635
+ """Prepare weight files for the model."""
636
+
637
+ allowed_patterns = ["*.safetensors", "*.bin", "*.pt"]
638
+
639
+ hf_weights_files, matched_pattern = self._get_weight_files(
640
+ model_name_or_path, allowed_patterns, revision)
641
+
642
+ if matched_pattern != "*.safetensors":
643
+ hf_weights_files = filter_files_not_needed_for_inference(
644
+ hf_weights_files)
645
+
646
+ if len(hf_weights_files) == 0:
647
+ raise RuntimeError(
648
+ f"Cannot find any model weights with `{model_name_or_path}`")
649
+
650
+ return hf_weights_files, matched_pattern == "*.safetensors"
651
+
652
+ def _get_quantized_weights_iterator(
653
+ self, model_name_or_path: str, revision: Optional[str]
654
+ ) -> Tuple[Generator[Tuple[str, torch.Tensor], None, None], Dict[str,
655
+ Any]]:
656
+ """Get an iterator to the model weights with bitsandbytes quantization,
657
+ as well as the quantization state dictionary."""
658
+
659
+ # only load the bitsandbytes module when needed
660
+ try:
661
+ import bitsandbytes
662
+ if bitsandbytes.__version__ < "0.42.0":
663
+ raise ImportError("bitsandbytes version is wrong. Please "
664
+ "install bitsandbytes>=0.42.0.")
665
+ from bitsandbytes.functional import quantize_4bit
666
+ except ImportError as err:
667
+ raise ImportError("Please install bitsandbytes>=0.42.0 via "
668
+ "`pip install bitsandbytes>=0.42.0` to use "
669
+ "bitsandbytes quantizer.") from err
670
+
671
+ hf_weights_files, use_safetensors = self._prepare_weights(
672
+ model_name_or_path, revision)
673
+
674
+ quant_state_dict = {}
675
+ if use_safetensors:
676
+ weight_iterator = safetensors_weights_iterator(hf_weights_files)
677
+ else:
678
+ weight_iterator = pt_weights_iterator(hf_weights_files)
679
+
680
+ def generator():
681
+ for weight_name, weight_tensor in weight_iterator:
682
+ if any(target_module in weight_name
683
+ for target_module in self.target_modules):
684
+ weight_name = weight_name.replace(".weight", ".qweight")
685
+ # bitsandbytes requires data in GPU
686
+ loaded_weight = weight_tensor.cuda().data
687
+ with set_default_torch_dtype(torch.float32):
688
+ processed_weight, quant_state = quantize_4bit(
689
+ loaded_weight,
690
+ compress_statistics=True,
691
+ quant_type="nf4")
692
+
693
+ quant_state_dict[weight_name] = quant_state
694
+ else:
695
+ processed_weight = weight_tensor
696
+
697
+ yield weight_name, processed_weight
698
+
699
+ return generator(), quant_state_dict
700
+
701
+ def _load_weights(self, model_config: ModelConfig,
702
+ model: nn.Module) -> None:
703
+ if not hasattr(model, 'load_weights'):
704
+ raise AttributeError(
705
+ "The required method 'load_weights' is not defined in class"
706
+ f" {type(self).__name__}.")
707
+
708
+ if not hasattr(model, 'bitsandbytes_stacked_params_mapping'):
709
+ raise AttributeError(
710
+ f"Model {type(self).__name__} does not support BitsAndBytes "
711
+ "quantization yet.")
712
+
713
+ logger.info("Loading weights with BitsAndBytes quantization. "
714
+ " May take a while ...")
715
+
716
+ qweight_iterator, quant_state_dict = (
717
+ self._get_quantized_weights_iterator(model_config.model,
718
+ model_config.revision))
719
+
720
+ model.load_weights(qweight_iterator)
721
+
722
+ param_dict = dict(model.named_parameters())
723
+ stacked_quant_state_dict: Dict[str, Dict[int, Any]] = {}
724
+ for quant_param_name in quant_state_dict:
725
+ non_stacked_param_name = quant_param_name
726
+
727
+ shard_index = 0
728
+ for shard_name, (
729
+ weight_name, index
730
+ ) in model.bitsandbytes_stacked_params_mapping.items():
731
+ if shard_name in quant_param_name:
732
+ shard_index = index
733
+ quant_param_name = quant_param_name.replace(
734
+ shard_name, weight_name)
735
+ break
736
+
737
+ if quant_param_name not in param_dict:
738
+ raise ValueError(
739
+ f"Parameter {quant_param_name} not found in the model.")
740
+
741
+ if quant_param_name not in stacked_quant_state_dict:
742
+ stacked_quant_state_dict[quant_param_name] = {}
743
+
744
+ stacked_quant_state_dict[quant_param_name][shard_index] = (
745
+ quant_state_dict[non_stacked_param_name])
746
+
747
+ # save quant_states and offsets as the attributes of the parameters
748
+ for param_name, param in param_dict.items():
749
+ if param_name in stacked_quant_state_dict:
750
+ quant_states = stacked_quant_state_dict[param_name]
751
+ set_weight_attrs(param, {"bnb_quant_state": quant_states})
752
+
753
+ pack_ratio = getattr(param, "pack_factor", -1)
754
+ if pack_ratio == -1:
755
+ raise ValueError(
756
+ f"pack_factor not set for parameter {param_name}.")
757
+
758
+ num_elements = [0] * len(quant_states)
759
+ for seq, quant_state in enumerate(quant_states.items()):
760
+ num_elements[seq] = math.prod(
761
+ quant_state[1].shape) // pack_ratio
762
+
763
+ offsets = np.concatenate(([0], np.cumsum(num_elements)))
764
+ set_weight_attrs(param, {"bnb_shard_offsets": offsets})
765
+
766
+ def load_model(self, *, model_config: ModelConfig,
767
+ device_config: DeviceConfig,
768
+ lora_config: Optional[LoRAConfig],
769
+ vision_language_config: Optional[VisionLanguageConfig],
770
+ parallel_config: ParallelConfig,
771
+ scheduler_config: SchedulerConfig,
772
+ cache_config: CacheConfig) -> nn.Module:
773
+ with set_default_torch_dtype(model_config.dtype):
774
+ with torch.device(device_config.device):
775
+ model = _initialize_model(model_config, self.load_config,
776
+ lora_config, vision_language_config,
777
+ cache_config)
778
+
779
+ self._load_weights(model_config, model)
780
+
781
+ return model.eval()
782
+
783
+
784
def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
785
"""Get a model loader based on the load format."""
786
@@ -554,4 +796,7 @@ def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
796
if load_config.load_format == LoadFormat.SHARDED_STATE:
797
return ShardedStateLoader(load_config)
798
799
+ if load_config.load_format == LoadFormat.BITSANDBYTES:
800
+ return BitsAndBytesModelLoader(load_config)
801
+
802
return DefaultModelLoader(load_config)
vllm/model_executor/model_loader/weight_utils.py
+15
-1
@@ -130,7 +130,17 @@ def get_quant_config(model_config: ModelConfig,
130
131
if hf_quant_config is not None:
132
return quant_cls.from_config(hf_quant_config)
133
- model_name_or_path = model_config.model
133
+ # In case of bitsandbytes/QLoRA, get quant config from the adapter model.
134
+ if model_config.quantization == "bitsandbytes":
135
+ if (not load_config.model_loader_extra_config
136
+ or "qlora_adapter_name_or_path"
137
+ not in load_config.model_loader_extra_config):
138
+ return quant_cls.from_config({"adapter_name_or_path": ""})
139
+ model_name_or_path = load_config.model_loader_extra_config[
140
+ "qlora_adapter_name_or_path"]
141
+
142
+ else:
143
+ model_name_or_path = model_config.model
144
is_local = os.path.isdir(model_name_or_path)
145
if not is_local:
146
# Download the config files.
@@ -169,6 +179,10 @@ def get_quant_config(model_config: ModelConfig,
179
quant_config_file = quant_config_files[0]
180
with open(quant_config_file, "r") as f:
181
config = json.load(f)
182
+
183
+ if model_config.quantization == "bitsandbytes":
184
+ config["adapter_name_or_path"] = model_name_or_path
185
+
186
return quant_cls.from_config(config)
187
188
vllm/model_executor/models/llama.py
+8
@@ -319,6 +319,14 @@ class LlamaForCausalLM(nn.Module):
319
"lm_head": "output_embeddings",
320
}
321
embedding_padding_modules = ["lm_head"]
322
+ bitsandbytes_stacked_params_mapping = {
323
+ # shard_name, weight_name, index
324
+ "q_proj": ("qkv_proj", 0),
325
+ "k_proj": ("qkv_proj", 1),
326
+ "v_proj": ("qkv_proj", 2),
327
+ "gate_proj": ("gate_up_proj", 0),
328
+ "up_proj": ("gate_up_proj", 1),
329
+ }
330
331
def __init__(
332
self,