@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b9c0605a8e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Feature][Kernel] Support bitsandbytes quantization and QLoRA (#4776)

chenqianfzh committed Jun 1, 2024 at 13:51 UTC b9c0605a8e7d558f595bd59ba6e6c95578dc0f1e
11 files changed +752 -8
examples/lora_with_quantization_inference.py new
+140
@@ -0,0 +1,140 @@
1 +"""
2 +This example shows how to use LoRA with different quantization techniques
3 +for offline inference.
4 +
5 +Requires HuggingFace credentials for access.
6 +"""
7 +
8 +import gc
9 +from typing import List, Optional, Tuple
10 +
11 +import torch
12 +from huggingface_hub import snapshot_download
13 +
14 +from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
15 +from vllm.lora.request import LoRARequest
16 +
17 +
18 +def create_test_prompts(
19 + lora_path: str
20 +) -> List[Tuple[str, SamplingParams, Optional[LoRARequest]]]:
21 + return [
22 + # this is an example of using quantization without LoRA
23 + ("My name is",
24 + SamplingParams(temperature=0.0,
25 + logprobs=1,
26 + prompt_logprobs=1,
27 + max_tokens=128), None),
28 + # the next three examples use quantization with LoRA
29 + ("my name is",
30 + SamplingParams(temperature=0.0,
31 + logprobs=1,
32 + prompt_logprobs=1,
33 + max_tokens=128),
34 + LoRARequest("lora-test-1", 1, lora_path)),
35 + ("The capital of USA is",
36 + SamplingParams(temperature=0.0,
37 + logprobs=1,
38 + prompt_logprobs=1,
39 + max_tokens=128),
40 + LoRARequest("lora-test-2", 1, lora_path)),
41 + ("The capital of France is",
42 + SamplingParams(temperature=0.0,
43 + logprobs=1,
44 + prompt_logprobs=1,
45 + max_tokens=128),
46 + LoRARequest("lora-test-3", 1, lora_path)),
47 + ]
48 +
49 +
50 +def process_requests(engine: LLMEngine,
51 + test_prompts: List[Tuple[str, SamplingParams,
52 + Optional[LoRARequest]]]):
53 + """Continuously process a list of prompts and handle the outputs."""
54 + request_id = 0
55 +
56 + while test_prompts or engine.has_unfinished_requests():
57 + if test_prompts:
58 + prompt, sampling_params, lora_request = test_prompts.pop(0)
59 + engine.add_request(str(request_id),
60 + prompt,
61 + sampling_params,
62 + lora_request=lora_request)
63 + request_id += 1
64 +
65 + request_outputs: List[RequestOutput] = engine.step()
66 + for request_output in request_outputs:
67 + if request_output.finished:
68 + print("----------------------------------------------------")
69 + print(f"Prompt: {request_output.prompt}")
70 + print(f"Output: {request_output.outputs[0].text}")
71 +
72 +
73 +def initialize_engine(model: str, quantization: str,
74 + lora_repo: Optional[str]) -> LLMEngine:
75 + """Initialize the LLMEngine."""
76 +
77 + if quantization == "bitsandbytes":
78 + # QLoRA (https://arxiv.org/abs/2305.14314) is a quantization technique.
79 + # It quantizes the model when loading, with some config info from the
80 + # LoRA adapter repo. So need to set the parameter of load_format and
81 + # qlora_adapter_name_or_path as below.
82 + engine_args = EngineArgs(
83 + model=model,
84 + quantization=quantization,
85 + qlora_adapter_name_or_path=lora_repo,
86 + load_format="bitsandbytes",
87 + enable_lora=True,
88 + max_lora_rank=64,
89 + # set it only in GPUs of limited memory
90 + enforce_eager=True)
91 + else:
92 + engine_args = EngineArgs(
93 + model=model,
94 + quantization=quantization,
95 + enable_lora=True,
96 + max_loras=4,
97 + # set it only in GPUs of limited memory
98 + enforce_eager=True)
99 + return LLMEngine.from_engine_args(engine_args)
100 +
101 +
102 +def main():
103 + """Main function that sets up and runs the prompt processing."""
104 +
105 + test_configs = [{
106 + "name": "qlora_inference_example",
107 + 'model': "huggyllama/llama-7b",
108 + 'quantization': "bitsandbytes",
109 + 'lora_repo': 'timdettmers/qlora-flan-7b'
110 + }, {
111 + "name": "AWQ_inference_with_lora_example",
112 + 'model': 'TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ',
113 + 'quantization': "awq",
114 + 'lora_repo': 'jashing/tinyllama-colorist-lora'
115 + }, {
116 + "name": "GPTQ_inference_with_lora_example",
117 + 'model': 'TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ',
118 + 'quantization': "gptq",
119 + 'lora_repo': 'jashing/tinyllama-colorist-lora'
120 + }]
121 +
122 + for test_config in test_configs:
123 + print(
124 + f"~~~~~~~~~~~~~~~~ Running: {test_config['name']} ~~~~~~~~~~~~~~~~"
125 + )
126 + engine = initialize_engine(test_config['model'],
127 + test_config['quantization'],
128 + test_config['lora_repo'])
129 + lora_path = snapshot_download(repo_id=test_config['lora_repo'])
130 + test_prompts = create_test_prompts(lora_path)
131 + process_requests(engine, test_prompts)
132 +
133 + # Clean up the GPU memory for the next test
134 + del engine
135 + gc.collect()
136 + torch.cuda.empty_cache()
137 +
138 +
139 +if __name__ == '__main__':
140 + main()
requirements-dev.txt
+3
@@ -35,3 +35,6 @@ aiohttp
35
36 # Multimodal
37 pillow
38 +
39 +# quantization
40 +bitsandbytes==0.42.0
tests/quantization/test_bitsandbytes.py new
+80
@@ -0,0 +1,80 @@
1 +'''Tests whether bitsandbytes computation is enabled correctly.
2 +
3 +Run `pytest tests/quantization/test_bitsandbytes.py`.
4 +'''
5 +import pytest
6 +import torch
7 +
8 +from vllm import SamplingParams
9 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
10 +
11 +capability = torch.cuda.get_device_capability()
12 +capability = capability[0] * 10 + capability[1]
13 +
14 +
15 +@pytest.mark.skipif(
16 + capability < QUANTIZATION_METHODS['bitsandbytes'].get_min_capability(),
17 + reason='bitsandbytes is not supported on this GPU type.')
18 +def test_load_bnb_model(vllm_runner) -> None:
19 + llm = vllm_runner('huggyllama/llama-7b',
20 + quantization='bitsandbytes',
21 + load_format='bitsandbytes',
22 + enforce_eager=True)
23 +
24 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
25 +
26 + # check the weights in MLP & SelfAttention are quantized to torch.uint8
27 + qweight = model.model.layers[0].mlp.gate_up_proj.qweight
28 + assert qweight.dtype == torch.uint8, (
29 + f'Expected gate_up_proj dtype torch.uint8 but got {qweight.dtype}')
30 +
31 + qweight = model.model.layers[0].mlp.down_proj.qweight
32 + assert qweight.dtype == torch.uint8, (
33 + f'Expected down_proj dtype torch.uint8 but got {qweight.dtype}')
34 +
35 + qweight = model.model.layers[0].self_attn.o_proj.qweight
36 + assert qweight.dtype == torch.uint8, (
37 + f'Expected o_proj dtype torch.uint8 but got {qweight.dtype}')
38 +
39 + qweight = model.model.layers[0].self_attn.qkv_proj.qweight
40 + assert qweight.dtype == torch.uint8, (
41 + f'Expected qkv_proj dtype torch.uint8 but got {qweight.dtype}')
42 +
43 + # some weights should not be quantized
44 + weight = model.lm_head.weight
45 + assert weight.dtype != torch.uint8, (
46 + 'lm_head weight dtype should not be torch.uint8')
47 +
48 + weight = model.model.embed_tokens.weight
49 + assert weight.dtype != torch.uint8, (
50 + 'embed_tokens weight dtype should not be torch.uint8')
51 +
52 + weight = model.model.layers[0].input_layernorm.weight
53 + assert weight.dtype != torch.uint8, (
54 + 'input_layernorm weight dtype should not be torch.uint8')
55 +
56 + weight = model.model.layers[0].post_attention_layernorm.weight
57 + assert weight.dtype != torch.uint8, (
58 + 'input_layernorm weight dtype should not be torch.uint8')
59 +
60 + # check the output of the model is expected
61 + sampling_params = SamplingParams(temperature=0.0,
62 + logprobs=1,
63 + prompt_logprobs=1,
64 + max_tokens=8)
65 +
66 + prompts = ['That which does not kill us', 'To be or not to be,']
67 + expected_outputs = [
68 + 'That which does not kill us makes us stronger.',
69 + 'To be or not to be, that is the question.'
70 + ]
71 + outputs = llm.generate(prompts, sampling_params=sampling_params)
72 +
73 + assert len(outputs) == len(prompts)
74 +
75 + for index in range(len(outputs)):
76 + # compare the first line of the output
77 + actual_output = outputs[index][1][0].split('\n', 1)[0]
78 + expected_output = expected_outputs[index].split('\n', 1)[0]
79 + assert actual_output == expected_output, (
80 + f'Expected: {expected_output}, but got: {actual_output}')
vllm/config.py
+8 -1
@@ -241,6 +241,12 @@ class ModelConfig:
241 "must be divisible by pipeline parallel size "
242 f"({pipeline_parallel_size}).")
243
244 + if self.quantization == "bitsandbytes" and (
245 + parallel_config.tensor_parallel_size > 1
246 + or parallel_config.pipeline_parallel_size > 1):
247 + raise ValueError(
248 + "BitAndBytes quantization with TP or PP is not supported yet.")
249 +
250 def get_hf_config_sliding_window(self) -> Optional[int]:
251 """Get the sliding window size, or None if disabled.
252 """
@@ -327,7 +333,7 @@ class ModelConfig:
333 def get_num_attention_heads(self,
334 parallel_config: "ParallelConfig") -> int:
335 return self.hf_text_config.num_attention_heads // \
330 - parallel_config.tensor_parallel_size
336 + parallel_config.tensor_parallel_size
337
338 def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
339 total_num_hidden_layers = self.hf_text_config.num_hidden_layers
@@ -487,6 +493,7 @@ class LoadFormat(str, enum.Enum):
493 DUMMY = "dummy"
494 TENSORIZER = "tensorizer"
495 SHARDED_STATE = "sharded_state"
496 + BITSANDBYTES = "bitsandbytes"
497
498
499 @dataclass
vllm/engine/arg_utils.py
+35 -3
@@ -92,6 +92,8 @@ class EngineArgs:
92 ngram_prompt_lookup_max: Optional[int] = None
93 ngram_prompt_lookup_min: Optional[int] = None
94
95 + qlora_adapter_name_or_path: Optional[str] = None
96 +
97 def __post_init__(self):
98 if self.tokenizer is None:
99 self.tokenizer = self.model
@@ -159,7 +161,8 @@ class EngineArgs:
161 type=str,
162 default=EngineArgs.load_format,
163 choices=[
162 - 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer'
164 + 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer',
165 + 'bitsandbytes'
166 ],
167 help='The format of the model weights to load.\n\n'
168 '* "auto" will try to load the weights in the safetensors format '
@@ -173,7 +176,9 @@ class EngineArgs:
176 'which is mainly for profiling.\n'
177 '* "tensorizer" will load the weights using tensorizer from '
178 'CoreWeave. See the Tensorize vLLM Model script in the Examples'
176 - 'section for more information.\n')
179 + 'section for more information.\n'
180 + '* "bitsandbytes" will load the weights using bitsandbytes '
181 + 'quantization.\n')
182 parser.add_argument(
183 '--dtype',
184 type=str,
@@ -543,7 +548,10 @@ class EngineArgs:
548 "will also be used in `model_name` tag content of "
549 "prometheus metrics, if multiple names provided, metrics"
550 "tag will take the first one.")
546 -
551 + parser.add_argument('--qlora-adapter-name-or-path',
552 + type=str,
553 + default=None,
554 + help='Name or path of the QLoRA adapter.')
555 return parser
556
557 @classmethod
@@ -555,6 +563,23 @@ class EngineArgs:
563 return engine_args
564
565 def create_engine_config(self, ) -> EngineConfig:
566 +
567 + # bitsandbytes quantization needs a specific model loader
568 + # so we make sure the quant method and the load format are consistent
569 + if (self.quantization == "bitsandbytes" or
570 + self.qlora_adapter_name_or_path is not None) and \
571 + self.load_format != "bitsandbytes":
572 + raise ValueError(
573 + "BitsAndBytes quantization and QLoRA adapter only support "
574 + f"'bitsandbytes' load format, but got {self.load_format}")
575 +
576 + if (self.load_format == "bitsandbytes" or
577 + self.qlora_adapter_name_or_path is not None) and \
578 + self.quantization != "bitsandbytes":
579 + raise ValueError(
580 + "BitsAndBytes load format and QLoRA adapter only support "
581 + f"'bitsandbytes' quantization, but got {self.quantization}")
582 +
583 device_config = DeviceConfig(self.device)
584 model_config = ModelConfig(
585 self.model, self.tokenizer, self.tokenizer_mode,
@@ -622,6 +647,13 @@ class EngineArgs:
647 max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
648 and self.max_cpu_loras > 0 else None) if self.enable_lora else None
649
650 + if self.qlora_adapter_name_or_path is not None and \
651 + self.qlora_adapter_name_or_path != "":
652 + if self.model_loader_extra_config is None:
653 + self.model_loader_extra_config = {}
654 + self.model_loader_extra_config[
655 + "qlora_adapter_name_or_path"] = self.qlora_adapter_name_or_path
656 +
657 load_config = LoadConfig(
658 load_format=self.load_format,
659 download_dir=self.download_dir,
vllm/model_executor/layers/linear.py
+39 -2
@@ -1,5 +1,5 @@
1 from abc import abstractmethod
2 -from typing import List, Optional
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5 import torch.nn.functional as F
@@ -26,6 +26,21 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
26 return shard_size * marlin_tile_size, shard_offset * marlin_tile_size
27
28
29 +def adjust_bitsandbytes_shard(param: Parameter,
30 + qkv_offsets: Dict[str, Tuple[int, int]],
31 + loaded_shard_id: str) -> Tuple[int, int]:
32 + """Adjust the quantization offsets and sizes for BitsAndBytes sharding."""
33 +
34 + total, _ = qkv_offsets["total"]
35 + orig_offset, orig_size = qkv_offsets[loaded_shard_id]
36 +
37 + quantized_total = param.data.shape[0]
38 + quantized_offset = orig_offset * quantized_total // total
39 + quantized_size = orig_size * quantized_total // total
40 +
41 + return quantized_size, quantized_offset
42 +
43 +
44 class LinearMethodBase(QuantizeMethodBase):
45 """Base class for different (maybe quantized) linear methods."""
46
@@ -37,7 +52,7 @@ class LinearMethodBase(QuantizeMethodBase):
52 **extra_weight_attrs):
53 """Create weights for a linear layer.
54 The weights will be set as attributes of the layer.
40 -
55 +
56 Args:
57 layer: The layer that is using the LinearMethodBase factory.
58 input_size_per_partition: Size of the weight input dim on rank X.
@@ -416,6 +431,12 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
431 shard_size, shard_offset = adjust_marlin_shard(
432 param, shard_size, shard_offset)
433
434 + use_bitsandbytes = getattr(param, "use_bitsandbytes", False)
435 + if use_bitsandbytes:
436 + shard_size = loaded_weight.shape[output_dim]
437 + shard_offset = loaded_weight.shape[output_dim] * \
438 + loaded_shard_id
439 +
440 param_data = param_data.narrow(output_dim, shard_offset,
441 shard_size)
442 start_idx = tp_rank * shard_size
@@ -615,6 +636,22 @@ class QKVParallelLinear(ColumnParallelLinear):
636 shard_size, shard_offset = adjust_marlin_shard(
637 param, shard_size, shard_offset)
638
639 + use_bitsandbytes = getattr(param, "use_bitsandbytes", False)
640 + if use_bitsandbytes:
641 + orig_qkv_offsets = {
642 + "q": (0, self.num_heads * self.head_size),
643 + "k": (self.num_heads * self.head_size,
644 + self.num_kv_heads * self.head_size),
645 + "v":
646 + ((self.num_heads + self.num_kv_heads) * self.head_size,
647 + self.num_kv_heads * self.head_size),
648 + "total":
649 + ((self.num_heads + 2 * self.num_kv_heads) * self.head_size,
650 + 0)
651 + }
652 + shard_size, shard_offset = adjust_bitsandbytes_shard(
653 + param, orig_qkv_offsets, loaded_shard_id)
654 +
655 param_data = param_data.narrow(output_dim, shard_offset,
656 shard_size)
657 if loaded_shard_id == "q":
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -4,6 +4,8 @@ from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4 from vllm.model_executor.layers.quantization.awq import AWQConfig
5 from vllm.model_executor.layers.quantization.base_config import (
6 QuantizationConfig)
7 +from vllm.model_executor.layers.quantization.bitsandbytes import (
8 + BitsAndBytesConfig)
9 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10 CompressedTensorsConfig)
11 from vllm.model_executor.layers.quantization.deepspeedfp import (
@@ -30,6 +32,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
32 "gptq": GPTQConfig,
33 "squeezellm": SqueezeLLMConfig,
34 "sparseml": CompressedTensorsConfig,
35 + "bitsandbytes": BitsAndBytesConfig,
36 }
37
38
vllm/model_executor/layers/quantization/bitsandbytes.py new
+175
@@ -0,0 +1,175 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +from torch.nn.parameter import Parameter
5 +
6 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
7 + set_weight_attrs)
8 +from vllm.model_executor.layers.quantization.base_config import (
9 + QuantizationConfig)
10 +
11 +
12 +class BitsAndBytesConfig(QuantizationConfig):
13 + """Config class for BitsAndBytes Quantization.
14 +
15 + Reference: https://arxiv.org/abs/2305.14314
16 + """
17 +
18 + def __init__(
19 + self,
20 + adapter_name_or_path: str,
21 + target_modules: List[str],
22 + ) -> None:
23 +
24 + self.adapter_name_or_path = adapter_name_or_path
25 + self.target_modules = target_modules
26 +
27 + def __repr__(self) -> str:
28 + return (
29 + f"BitsAndBytesConfig(adapter_name_or_path={self.adapter_name_or_path}"
30 + )
31 +
32 + @classmethod
33 + def get_name(self) -> str:
34 + return "bitsandbytes"
35 +
36 + @classmethod
37 + def get_supported_act_dtypes(self) -> List[torch.dtype]:
38 + return [torch.float32, torch.float16, torch.bfloat16]
39 +
40 + @classmethod
41 + def get_min_capability(self) -> int:
42 + return 70
43 +
44 + @staticmethod
45 + def get_config_filenames() -> List[str]:
46 + return [
47 + "adapter_config.json",
48 + ]
49 +
50 + @classmethod
51 + def from_config(cls, config: Dict[str, Any]) -> "BitsAndBytesConfig":
52 + adapter_name = cls.get_from_keys(config, ["adapter_name_or_path"])
53 + default_target_modules = [
54 + "gate_proj", "down_proj", "up_proj", "q_proj", "k_proj", "v_proj",
55 + "o_proj"
56 + ]
57 + if adapter_name == "":
58 + target_modules = default_target_modules
59 + else:
60 + target_modules = cls.get_from_keys(config, ["target_modules"])
61 + return cls(adapter_name, target_modules)
62 +
63 + def get_quant_method(
64 + self,
65 + layer: torch.nn.Module) -> Optional["BitsAndBytesLinearMethod"]:
66 + if isinstance(layer, LinearBase):
67 + return BitsAndBytesLinearMethod(self)
68 + return None
69 +
70 + def get_scaled_act_names(self) -> List[str]:
71 + return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
72 +
73 +
74 +class BitsAndBytesLinearMethod(LinearMethodBase):
75 + """Linear method for BitsAndBytes.
76 +
77 + Args:
78 + quant_config: The BitsAndBytes quantization config.
79 + """
80 +
81 + def __init__(self, quant_config: BitsAndBytesConfig):
82 + try:
83 + import bitsandbytes
84 + if bitsandbytes.__version__ < "0.42.0":
85 + raise ImportError("bitsandbytes version is wrong. Please "
86 + "install bitsandbytes>=0.42.0.")
87 + except ImportError as err:
88 + raise ImportError("Please install bitsandbytes>=0.42.0 via "
89 + "`pip install bitsandbytes>=0.42.0` to use "
90 + "bitsandbytes quantizer.") from err
91 +
92 + self.quant_config = quant_config
93 +
94 + def create_weights(self, layer: torch.nn.Module,
95 + input_size_per_partition: int,
96 + output_partition_sizes: List[int], input_size: int,
97 + output_size: int, params_dtype: torch.dtype,
98 + **extra_weight_attrs):
99 + quant_ratio = 0
100 + if params_dtype.is_floating_point:
101 + quant_ratio = torch.finfo(params_dtype).bits // torch.iinfo(
102 + torch.uint8).bits
103 + else:
104 + quant_ratio = torch.iinfo(params_dtype).bits // torch.iinfo(
105 + torch.uint8).bits
106 +
107 + if input_size_per_partition * sum(
108 + output_partition_sizes) % quant_ratio != 0:
109 + raise ValueError(
110 + "The input size is not aligned with the quantized "
111 + "weight shape. ")
112 + qweight = Parameter(
113 + torch.empty(
114 + input_size_per_partition * sum(output_partition_sizes) //
115 + quant_ratio,
116 + 1,
117 + dtype=torch.uint8,
118 + ),
119 + requires_grad=False,
120 + )
121 +
122 + set_weight_attrs(
123 + qweight,
124 + {
125 + "input_dim": 0,
126 + # In bitsandbytes, a tensor of shape [n,m] is quantized to
127 + #[n*m/pack_ratio, 1],so the output_dim is 0
128 + "output_dim": 0,
129 + "pack_factor": quant_ratio,
130 + "use_bitsandbytes": True,
131 + })
132 + layer.register_parameter("qweight", qweight)
133 + set_weight_attrs(qweight, extra_weight_attrs)
134 +
135 + def apply(self,
136 + layer: torch.nn.Module,
137 + x: torch.Tensor,
138 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
139 +
140 + # only load the bitsandbytes module when needed
141 + from bitsandbytes import matmul_4bit
142 +
143 + original_type = x.dtype
144 + bf_x = x.to(torch.bfloat16)
145 +
146 + qweight = layer.qweight
147 + quant_states = qweight.bnb_quant_state
148 + offsets = qweight.bnb_shard_offsets
149 +
150 + out_dim_0 = x.shape[0]
151 + out_dim_1 = sum(
152 + [quant_state[1].shape[0] for quant_state in quant_states.items()])
153 + out = torch.empty(out_dim_0,
154 + out_dim_1,
155 + dtype=torch.bfloat16,
156 + device=x.device)
157 +
158 + current_index = 0
159 + for i in range(len(quant_states)):
160 + output_size = quant_states[i].shape[0]
161 + # It is more efficient to use out kwarg like
162 + # matmul_4bit(..., out = ...). Infeasible now due to the bug
163 + # https://github.com/TimDettmers/bitsandbytes/issues/1235.
164 + # Need to change after the bug is fixed.
165 + out[:, current_index:current_index + output_size] = matmul_4bit(
166 + bf_x, qweight[offsets[i]:offsets[i + 1]].t(), quant_states[i])
167 +
168 + current_index += output_size
169 +
170 + out = out.to(original_type)
171 +
172 + if bias is not None:
173 + out += bias
174 +
175 + return out
vllm/model_executor/model_loader/loader.py
+246 -1
@@ -1,13 +1,18 @@
1 # ruff: noqa: SIM117
2 import collections
3 import copy
4 +import fnmatch
5 import glob
6 +import json
7 +import math
8 import os
9 from abc import ABC, abstractmethod
10 from typing import Any, Dict, Generator, List, Optional, Tuple, Type
11
12 import huggingface_hub
13 +import numpy as np
14 import torch
15 +from huggingface_hub import HfApi, hf_hub_download
16 from torch import nn
17
18 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
@@ -28,6 +33,7 @@ from vllm.model_executor.model_loader.weight_utils import (
33 get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
34 pt_weights_iterator, safetensors_weights_iterator)
35 from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
36 +from vllm.model_executor.utils import set_weight_attrs
37
38 logger = init_logger(__name__)
39
@@ -125,7 +131,7 @@ class DefaultModelLoader(BaseModelLoader):
131 def _maybe_download_from_modelscope(
132 self, model: str, revision: Optional[str]) -> Optional[str]:
133 """Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True.
128 -
134 +
135 Returns the path to the downloaded model, or None if the model is not
136 downloaded from ModelScope."""
137 if VLLM_USE_MODELSCOPE:
@@ -247,6 +253,7 @@ class DefaultModelLoader(BaseModelLoader):
253 model,
254 "fall_back_to_pt_during_load",
255 True)), )
256 +
257 for _, module in model.named_modules():
258 quant_method = getattr(module, "quant_method", None)
259 if quant_method is not None:
@@ -539,6 +546,241 @@ class ShardedStateLoader(BaseModelLoader):
546 )
547
548
549 +class BitsAndBytesModelLoader(BaseModelLoader):
550 + """Model loader to load model weights with BitAndBytes quantization."""
551 +
552 + default_target_modules = [
553 + "gate_proj", "down_proj", "up_proj", "q_proj", "k_proj", "v_proj",
554 + "o_proj"
555 + ]
556 +
557 + possible_config_file_names = ["adapter_config.json"]
558 +
559 + def __init__(self, load_config: LoadConfig):
560 + super().__init__(load_config)
561 +
562 + # we don't need to quantize the whole model, only the target modules
563 + # that are specified in the adapter config file. If the adapter config
564 + # file is not provided, we will quantize the default modules.
565 + if (not load_config.model_loader_extra_config
566 + or "qlora_adapter_name_or_path"
567 + not in load_config.model_loader_extra_config):
568 + self.target_modules = self.default_target_modules
569 + return
570 +
571 + qlora_adapter = load_config.model_loader_extra_config[
572 + "qlora_adapter_name_or_path"]
573 +
574 + config_file_path = self._get_config_file(qlora_adapter)
575 +
576 + with open(config_file_path, "r") as f:
577 + config = json.load(f)
578 + self.target_modules = config["target_modules"]
579 +
580 + def _get_config_file(self, qlora_adapter: str) -> str:
581 + is_local = os.path.isdir(qlora_adapter)
582 + config_file_path = None
583 + if is_local:
584 + for file in self.possible_config_file_names:
585 + config_file_path = os.path.join(qlora_adapter, file)
586 + if os.path.exists(config_file_path):
587 + break
588 + else:
589 + hf_api = HfApi()
590 + repo_files = hf_api.list_repo_files(repo_id=qlora_adapter)
591 + for file in self.possible_config_file_names:
592 + if file in repo_files:
593 + config_file_path = hf_hub_download(repo_id=qlora_adapter,
594 + filename=file)
595 + break
596 +
597 + if not config_file_path:
598 + raise ValueError(
599 + f"Cannot find adapter config file in {qlora_adapter}")
600 +
601 + return config_file_path
602 +
603 + def _get_weight_files(
604 + self,
605 + model_name_or_path: str,
606 + allowed_patterns: List[str],
607 + revision: Optional[str] = None) -> Tuple[List[str], str]:
608 + """Retrieve weight files. Download the files if necessary.
609 +
610 + Return the weight files and the file pattern."""
611 + is_local = os.path.isdir(model_name_or_path)
612 +
613 + if is_local:
614 + for pattern in allowed_patterns:
615 + weight_files = glob.glob(
616 + os.path.join(model_name_or_path, pattern))
617 + if weight_files:
618 + return weight_files, pattern
619 + else:
620 + hf_api = HfApi()
621 + repo_files = hf_api.list_repo_files(repo_id=model_name_or_path)
622 + for pattern in allowed_patterns:
623 + matching_files = fnmatch.filter(repo_files, pattern)
624 + if matching_files:
625 + hf_folder = download_weights_from_hf(
626 + model_name_or_path, self.load_config.download_dir,
627 + [pattern], revision)
628 + return glob.glob(os.path.join(hf_folder, pattern)), pattern
629 +
630 + raise RuntimeError(
631 + f"No model weights found in: `{model_name_or_path}`")
632 +
633 + def _prepare_weights(self, model_name_or_path: str,
634 + revision: Optional[str]) -> Tuple[List[str], bool]:
635 + """Prepare weight files for the model."""
636 +
637 + allowed_patterns = ["*.safetensors", "*.bin", "*.pt"]
638 +
639 + hf_weights_files, matched_pattern = self._get_weight_files(
640 + model_name_or_path, allowed_patterns, revision)
641 +
642 + if matched_pattern != "*.safetensors":
643 + hf_weights_files = filter_files_not_needed_for_inference(
644 + hf_weights_files)
645 +
646 + if len(hf_weights_files) == 0:
647 + raise RuntimeError(
648 + f"Cannot find any model weights with `{model_name_or_path}`")
649 +
650 + return hf_weights_files, matched_pattern == "*.safetensors"
651 +
652 + def _get_quantized_weights_iterator(
653 + self, model_name_or_path: str, revision: Optional[str]
654 + ) -> Tuple[Generator[Tuple[str, torch.Tensor], None, None], Dict[str,
655 + Any]]:
656 + """Get an iterator to the model weights with bitsandbytes quantization,
657 + as well as the quantization state dictionary."""
658 +
659 + # only load the bitsandbytes module when needed
660 + try:
661 + import bitsandbytes
662 + if bitsandbytes.__version__ < "0.42.0":
663 + raise ImportError("bitsandbytes version is wrong. Please "
664 + "install bitsandbytes>=0.42.0.")
665 + from bitsandbytes.functional import quantize_4bit
666 + except ImportError as err:
667 + raise ImportError("Please install bitsandbytes>=0.42.0 via "
668 + "`pip install bitsandbytes>=0.42.0` to use "
669 + "bitsandbytes quantizer.") from err
670 +
671 + hf_weights_files, use_safetensors = self._prepare_weights(
672 + model_name_or_path, revision)
673 +
674 + quant_state_dict = {}
675 + if use_safetensors:
676 + weight_iterator = safetensors_weights_iterator(hf_weights_files)
677 + else:
678 + weight_iterator = pt_weights_iterator(hf_weights_files)
679 +
680 + def generator():
681 + for weight_name, weight_tensor in weight_iterator:
682 + if any(target_module in weight_name
683 + for target_module in self.target_modules):
684 + weight_name = weight_name.replace(".weight", ".qweight")
685 + # bitsandbytes requires data in GPU
686 + loaded_weight = weight_tensor.cuda().data
687 + with set_default_torch_dtype(torch.float32):
688 + processed_weight, quant_state = quantize_4bit(
689 + loaded_weight,
690 + compress_statistics=True,
691 + quant_type="nf4")
692 +
693 + quant_state_dict[weight_name] = quant_state
694 + else:
695 + processed_weight = weight_tensor
696 +
697 + yield weight_name, processed_weight
698 +
699 + return generator(), quant_state_dict
700 +
701 + def _load_weights(self, model_config: ModelConfig,
702 + model: nn.Module) -> None:
703 + if not hasattr(model, 'load_weights'):
704 + raise AttributeError(
705 + "The required method 'load_weights' is not defined in class"
706 + f" {type(self).__name__}.")
707 +
708 + if not hasattr(model, 'bitsandbytes_stacked_params_mapping'):
709 + raise AttributeError(
710 + f"Model {type(self).__name__} does not support BitsAndBytes "
711 + "quantization yet.")
712 +
713 + logger.info("Loading weights with BitsAndBytes quantization. "
714 + " May take a while ...")
715 +
716 + qweight_iterator, quant_state_dict = (
717 + self._get_quantized_weights_iterator(model_config.model,
718 + model_config.revision))
719 +
720 + model.load_weights(qweight_iterator)
721 +
722 + param_dict = dict(model.named_parameters())
723 + stacked_quant_state_dict: Dict[str, Dict[int, Any]] = {}
724 + for quant_param_name in quant_state_dict:
725 + non_stacked_param_name = quant_param_name
726 +
727 + shard_index = 0
728 + for shard_name, (
729 + weight_name, index
730 + ) in model.bitsandbytes_stacked_params_mapping.items():
731 + if shard_name in quant_param_name:
732 + shard_index = index
733 + quant_param_name = quant_param_name.replace(
734 + shard_name, weight_name)
735 + break
736 +
737 + if quant_param_name not in param_dict:
738 + raise ValueError(
739 + f"Parameter {quant_param_name} not found in the model.")
740 +
741 + if quant_param_name not in stacked_quant_state_dict:
742 + stacked_quant_state_dict[quant_param_name] = {}
743 +
744 + stacked_quant_state_dict[quant_param_name][shard_index] = (
745 + quant_state_dict[non_stacked_param_name])
746 +
747 + # save quant_states and offsets as the attributes of the parameters
748 + for param_name, param in param_dict.items():
749 + if param_name in stacked_quant_state_dict:
750 + quant_states = stacked_quant_state_dict[param_name]
751 + set_weight_attrs(param, {"bnb_quant_state": quant_states})
752 +
753 + pack_ratio = getattr(param, "pack_factor", -1)
754 + if pack_ratio == -1:
755 + raise ValueError(
756 + f"pack_factor not set for parameter {param_name}.")
757 +
758 + num_elements = [0] * len(quant_states)
759 + for seq, quant_state in enumerate(quant_states.items()):
760 + num_elements[seq] = math.prod(
761 + quant_state[1].shape) // pack_ratio
762 +
763 + offsets = np.concatenate(([0], np.cumsum(num_elements)))
764 + set_weight_attrs(param, {"bnb_shard_offsets": offsets})
765 +
766 + def load_model(self, *, model_config: ModelConfig,
767 + device_config: DeviceConfig,
768 + lora_config: Optional[LoRAConfig],
769 + vision_language_config: Optional[VisionLanguageConfig],
770 + parallel_config: ParallelConfig,
771 + scheduler_config: SchedulerConfig,
772 + cache_config: CacheConfig) -> nn.Module:
773 + with set_default_torch_dtype(model_config.dtype):
774 + with torch.device(device_config.device):
775 + model = _initialize_model(model_config, self.load_config,
776 + lora_config, vision_language_config,
777 + cache_config)
778 +
779 + self._load_weights(model_config, model)
780 +
781 + return model.eval()
782 +
783 +
784 def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
785 """Get a model loader based on the load format."""
786
@@ -554,4 +796,7 @@ def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
796 if load_config.load_format == LoadFormat.SHARDED_STATE:
797 return ShardedStateLoader(load_config)
798
799 + if load_config.load_format == LoadFormat.BITSANDBYTES:
800 + return BitsAndBytesModelLoader(load_config)
801 +
802 return DefaultModelLoader(load_config)
vllm/model_executor/model_loader/weight_utils.py
+15 -1
@@ -130,7 +130,17 @@ def get_quant_config(model_config: ModelConfig,
130
131 if hf_quant_config is not None:
132 return quant_cls.from_config(hf_quant_config)
133 - model_name_or_path = model_config.model
133 + # In case of bitsandbytes/QLoRA, get quant config from the adapter model.
134 + if model_config.quantization == "bitsandbytes":
135 + if (not load_config.model_loader_extra_config
136 + or "qlora_adapter_name_or_path"
137 + not in load_config.model_loader_extra_config):
138 + return quant_cls.from_config({"adapter_name_or_path": ""})
139 + model_name_or_path = load_config.model_loader_extra_config[
140 + "qlora_adapter_name_or_path"]
141 +
142 + else:
143 + model_name_or_path = model_config.model
144 is_local = os.path.isdir(model_name_or_path)
145 if not is_local:
146 # Download the config files.
@@ -169,6 +179,10 @@ def get_quant_config(model_config: ModelConfig,
179 quant_config_file = quant_config_files[0]
180 with open(quant_config_file, "r") as f:
181 config = json.load(f)
182 +
183 + if model_config.quantization == "bitsandbytes":
184 + config["adapter_name_or_path"] = model_name_or_path
185 +
186 return quant_cls.from_config(config)
187
188
vllm/model_executor/models/llama.py
+8
@@ -319,6 +319,14 @@ class LlamaForCausalLM(nn.Module):
319 "lm_head": "output_embeddings",
320 }
321 embedding_padding_modules = ["lm_head"]
322 + bitsandbytes_stacked_params_mapping = {
323 + # shard_name, weight_name, index
324 + "q_proj": ("qkv_proj", 0),
325 + "k_proj": ("qkv_proj", 1),
326 + "v_proj": ("qkv_proj", 2),
327 + "gate_proj": ("gate_up_proj", 0),
328 + "up_proj": ("gate_up_proj", 1),
329 + }
330
331 def __init__(
332 self,