@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9855b99502
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Feature][kernel] tensor parallelism with bitsandbytes quantization (#8434)

chenqianfzh committed Sep 17, 2024 at 08:09 UTC 9855b99502c7537db5ef018129e603650800ac46
4 files changed +80 -17
tests/quantization/test_bitsandbytes.py
+21 -5
@@ -64,6 +64,24 @@ def test_load_8bit_bnb_model(hf_runner, vllm_runner, example_prompts,
64 model_name)
65
66
67 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
68 + reason='Test requires at least 2 GPUs.')
69 +@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
70 + reason='bitsandbytes is not supported on this GPU type.')
71 +@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
72 +@fork_new_process_for_each_test
73 +def test_load_tp_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
74 + model_name, description) -> None:
75 +
76 + hf_model_kwargs = {"load_in_4bit": True}
77 + validate_generated_texts(hf_runner,
78 + vllm_runner,
79 + example_prompts[:1],
80 + model_name,
81 + hf_model_kwargs,
82 + vllm_tp_size=2)
83 +
84 +
85 def log_generated_texts(prompts, outputs, runner_name):
86 logged_texts = []
87 for i, (_, generated_text) in enumerate(outputs):
@@ -80,22 +98,21 @@ def validate_generated_texts(hf_runner,
98 vllm_runner,
99 prompts,
100 model_name,
83 - hf_model_kwargs=None):
101 + hf_model_kwargs=None,
102 + vllm_tp_size=1):
103
104 # NOTE: run vLLM first, as it requires a clean process
105 # when using distributed inference
87 -
88 - #Run with vLLM runner
106 with vllm_runner(model_name,
107 quantization='bitsandbytes',
108 load_format='bitsandbytes',
109 + tensor_parallel_size=vllm_tp_size,
110 enforce_eager=True,
111 gpu_memory_utilization=0.8) as llm:
112 vllm_outputs = llm.generate_greedy(prompts, 8)
113 vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")
114
115 # Clean up the GPU memory for the next test
98 - torch.cuda.synchronize()
116 gc.collect()
117 torch.cuda.empty_cache()
118
@@ -108,7 +125,6 @@ def validate_generated_texts(hf_runner,
125 hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")
126
127 # Clean up the GPU memory for the next test
111 - torch.cuda.synchronize()
128 gc.collect()
129 torch.cuda.empty_cache()
130
vllm/config.py
-6
@@ -393,12 +393,6 @@ class ModelConfig:
393 "Pipeline parallelism is only supported for the following "
394 f" architectures: {_PP_SUPPORTED_MODELS}.")
395
396 - if self.quantization == "bitsandbytes" and (
397 - parallel_config.tensor_parallel_size > 1
398 - or parallel_config.pipeline_parallel_size > 1):
399 - raise ValueError(
400 - "BitAndBytes quantization with TP or PP is not supported yet.")
401 -
396 # Remove the constraint after the bitsandbytes issue is fixed:
397 # https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1308
398 if self.quantization == "bitsandbytes" and self.enforce_eager is False:
vllm/model_executor/layers/linear.py
+16 -5
@@ -530,8 +530,11 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
530 param_data = param_data.narrow(output_dim, shard_offset,
531 shard_size)
532 start_idx = tp_rank * shard_size
533 - loaded_weight = loaded_weight.narrow(output_dim, start_idx,
534 - shard_size)
533 + # bitsandbytes loads the weights of the specific portion
534 + # no need to narrow here
535 + if not use_bitsandbytes_4bit:
536 + loaded_weight = loaded_weight.narrow(output_dim, start_idx,
537 + shard_size)
538 # Special case for AQLM codebooks.
539 elif is_metadata:
540 # metadata indicates fixed size concatenated along dim 0
@@ -899,8 +902,13 @@ class QKVParallelLinear(ColumnParallelLinear):
902 else:
903 shard_id = tp_rank // self.num_kv_head_replicas
904 start_idx = shard_id * shard_size
902 - loaded_weight = loaded_weight.narrow(output_dim, start_idx,
903 - shard_size)
905 +
906 + # bitsandbytes loads the weights of the specific portion
907 + # no need to narrow here
908 + if not use_bitsandbytes_4bit:
909 + loaded_weight = loaded_weight.narrow(output_dim, start_idx,
910 + shard_size)
911 +
912 # Special case for for AQLM codebooks.
913 elif is_metadata:
914 # metadata indicates fixed size concatenated along dim 0
@@ -1000,6 +1008,7 @@ class RowParallelLinear(LinearBase):
1008 tp_rank = get_tensor_model_parallel_rank()
1009 tp_size = get_tensor_model_parallel_world_size()
1010 input_dim = getattr(param, "input_dim", None)
1011 + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
1012
1013 # Special case for GGUF
1014 is_gguf_weight = getattr(param, "is_gguf_weight", False)
@@ -1015,7 +1024,9 @@ class RowParallelLinear(LinearBase):
1024 param.materialize(tuple(weight_shape), dtype=loaded_weight.dtype)
1025
1026 param_data = param.data
1018 - if input_dim is not None:
1027 + # bitsandbytes loads the weights of the specific portion
1028 + # no need to narrow here
1029 + if input_dim is not None and not use_bitsandbytes_4bit:
1030 shard_size = param_data.shape[input_dim]
1031 start_idx = tp_rank * shard_size
1032 loaded_weight = loaded_weight.narrow(input_dim, start_idx,
vllm/model_executor/model_loader/loader.py
+43 -1
@@ -22,6 +22,8 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
22 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
23 LoRAConfig, ModelConfig, MultiModalConfig,
24 ParallelConfig, SchedulerConfig)
25 +from vllm.distributed import (get_tensor_model_parallel_rank,
26 + get_tensor_model_parallel_world_size)
27 from vllm.envs import VLLM_USE_MODELSCOPE
28 from vllm.logger import init_logger
29 from vllm.model_executor.layers.quantization.base_config import (
@@ -689,6 +691,8 @@ class ShardedStateLoader(BaseModelLoader):
691 class BitsAndBytesModelLoader(BaseModelLoader):
692 """Model loader to load model weights with BitAndBytes quantization."""
693
694 + # TODO: these module names are for Llama only,
695 + # change so that it works with other models as well
696 default_target_modules = [
697 "gate_proj", "down_proj", "up_proj", "q_proj", "k_proj", "v_proj",
698 "o_proj"
@@ -911,13 +915,44 @@ class BitsAndBytesModelLoader(BaseModelLoader):
915 def _unquantized_generator(self, hf_weights_files, use_safetensors,
916 quant_state_dict) -> Generator:
917 from bitsandbytes.functional import quantize_4bit
918 + tp_size = get_tensor_model_parallel_world_size()
919 + tp_rank = get_tensor_model_parallel_rank()
920 +
921 for weight_name, weight_tensor in self._hf_weight_iter(
922 hf_weights_files, use_safetensors):
923 if any(target_module in weight_name
924 for target_module in self.target_modules):
925 weight_name = weight_name.replace(".weight", ".qweight")
926 +
927 + # weight partitions of different modules occur at
928 + # different dimensions
929 + # TODO: these module names are for Llama only,
930 + # change so that it works with other models as well
931 + if 'down_proj' in weight_name or 'o_proj' in weight_name:
932 + total_size = weight_tensor.size(-1)
933 + start_index = total_size // tp_size * tp_rank
934 + end_index = total_size // tp_size * (tp_rank + 1)
935 + weight_sub_tensor = weight_tensor[...,
936 + start_index:end_index]
937 +
938 + else:
939 + total_size = weight_tensor.size(0)
940 + start_index = total_size // tp_size * tp_rank
941 + end_index = total_size // tp_size * (tp_rank + 1)
942 + weight_sub_tensor = weight_tensor[start_index:end_index,
943 + ...]
944 +
945 # bitsandbytes requires data in GPU
920 - loaded_weight = weight_tensor.cuda().data
946 + if weight_sub_tensor.is_cuda:
947 + loaded_weight = weight_sub_tensor
948 + else:
949 + loaded_weight = weight_sub_tensor.cuda()
950 +
951 + # remove the following after the issue is fixed:
952 + # https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1342
953 + if loaded_weight.is_contiguous() is False:
954 + loaded_weight = loaded_weight.contiguous()
955 +
956 with set_default_torch_dtype(torch.float32):
957 processed_weight, quant_state = quantize_4bit(
958 loaded_weight,
@@ -958,6 +993,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
993 f"BitsAndBytes loader does not support {quant_method} "
994 "quantization")
995
996 + # The quant_states in pre_quantized models cannot work with a split
997 + # weight tensor. So TP does not work with pre_quantized bnb models.
998 + if pre_quant and get_tensor_model_parallel_world_size() > 1:
999 + raise ValueError(
1000 + "Prequant BitsAndBytes models with TP is not supported."
1001 + "Please try with PP.")
1002 +
1003 load_8bit = False
1004 if pre_quant:
1005 load_8bit = quant_config.get('load_in_8bit', False)