A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc]Reduce BNB static variable (#9987)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 5, 2024 at 01:04 UTC
fb2716d64117aaa6c36b97b09765aa10a89e2fe5
10 files changed
+20
-46
vllm/model_executor/model_loader/loader.py
+20
-20
@@ -28,7 +28,8 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
28
get_tensor_model_parallel_world_size)
29
from vllm.envs import VLLM_USE_MODELSCOPE
30
from vllm.logger import init_logger
31
-from vllm.model_executor.layers.linear import ReplicatedLinear
31
+from vllm.model_executor.layers.linear import (ReplicatedLinear,
32
+ RowParallelLinear)
33
from vllm.model_executor.layers.quantization.base_config import (
34
QuantizationConfig)
35
from vllm.model_executor.model_loader.tensorizer import (
@@ -727,6 +728,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
728
def __init__(self, load_config: LoadConfig):
729
super().__init__(load_config)
730
731
+ # Save the module names without sharding.
732
+ self.unsharded_weights_modules: List[str] = []
733
+ # Save the module names that are sharded by column.
734
+ self.column_sharded_weights_modules: List[str] = []
735
# we don't need to quantize the whole model, only the target modules
736
# that are specified in the adapter config file. If the adapter config
737
# file is not provided, we will quantize the default modules.
@@ -744,8 +749,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
749
with open(config_file_path, "r") as f:
750
config = json.load(f)
751
self.target_modules = config["target_modules"]
747
- # Save the module names without sharding.
748
- self.unsharded_weights_modules: List[str] = []
752
753
def _get_config_file(self, qlora_adapter: str) -> str:
754
is_local = os.path.isdir(qlora_adapter)
@@ -971,9 +974,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
974
for module in self.unsharded_weights_modules):
975
weight_sub_tensor = weight_tensor
976
# Shard by column
974
- elif any(module in weight_name
975
- for module in self.column_parallel_weights_modules):
976
-
977
+ elif any(
978
+ weight_name.startswith(module)
979
+ for module in self.column_sharded_weights_modules):
980
total_size = weight_tensor.size(-1)
981
start_index = total_size // tp_size * tp_rank
982
end_index = total_size // tp_size * (tp_rank + 1)
@@ -1028,20 +1031,17 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1031
else:
1032
self.target_modules = self.default_target_modules
1033
1031
- if hasattr(model, 'column_parallel_weights_modules'):
1032
- self.column_parallel_weights_modules = \
1033
- model.column_parallel_weights_modules
1034
- else:
1035
- self.column_parallel_weights_modules = []
1036
- # Some modules like `ReplicatedLinear` should not have their weights
1037
- # sharded. The reason for implementing it this way is to avoid new
1038
- # static variable in the model implementation.
1039
- # TODO: Can we reduce the static variables needed for BNB based on
1040
- # model information?
1041
- self.unsharded_weights_modules = [
1042
- name for name, module in model.named_modules()
1043
- if isinstance(module, (ReplicatedLinear, ))
1044
- ]
1034
+ for name, module in model.named_modules():
1035
+ # Some modules like `ReplicatedLinear` should not have their weights
1036
+ # sharded. The reason for implementing it this way is to avoid new
1037
+ # static variable in the model implementation.
1038
+ if isinstance(module, (ReplicatedLinear, )):
1039
+ self.unsharded_weights_modules.append(name)
1040
+ # In TP, these weights are partitioned along the column
1041
+ # dimension (dim=-1)
1042
+ elif isinstance(module, (RowParallelLinear, )):
1043
+ self.column_sharded_weights_modules.append(name)
1044
+
1045
self.model_type = type(model).__name__
1046
1047
logger.info("Loading weights with BitsAndBytes quantization. "
vllm/model_executor/models/falcon.py
-2
@@ -401,8 +401,6 @@ class FalconForCausalLM(nn.Module, SupportsPP):
401
".dense_h_to_4h.",
402
".dense_4h_to_h.",
403
]
404
- # in TP, these weights are partitioned along the column dimension (dim=-1)
405
- column_parallel_weights_modules = [".dense_4h_to_h.", ".dense."]
404
405
def __init__(
406
self,
vllm/model_executor/models/gemma.py
-3
@@ -350,7 +350,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
350
"gate_up_proj",
351
"down_proj",
352
]
353
-
353
# BitandBytes specific attributes
354
default_bitsandbytes_target_modules = [
355
".gate_proj.",
@@ -361,8 +360,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
360
".v_proj.",
361
".o_proj.",
362
]
364
- # in TP, these weights are partitioned along the column dimension (dim=-1)
365
- column_parallel_weights_modules = [".down_proj.", ".o_proj."]
363
bitsandbytes_stacked_params_mapping = {
364
# shard_name, weight_name, index
365
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/gemma2.py
-2
@@ -390,8 +390,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
390
".v_proj.",
391
".o_proj.",
392
]
393
- # in TP, these weights are partitioned along the column dimension (dim=-1)
394
- column_parallel_weights_modules = [".down_proj.", ".o_proj."]
393
bitsandbytes_stacked_params_mapping = {
394
# shard_name, weight_name, index
395
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/llama.py
-2
@@ -464,8 +464,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
464
".v_proj.",
465
".o_proj.",
466
]
467
- # in TP, these weights are partitioned along the column dimension (dim=-1)
468
- column_parallel_weights_modules = [".down_proj.", ".o_proj."]
467
bitsandbytes_stacked_params_mapping = {
468
# shard_name, weight_name, index
469
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/minicpmv.py
-8
@@ -854,10 +854,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
854
# resampler
855
".kv_proj.",
856
]
857
- # in TP, these weights are partitioned along the column dimension (dim=-1)
858
- column_parallel_weights_modules = [
859
- ".down_proj.", ".o_proj.", ".self_attn.out_proj.", ".fc2."
860
- ]
857
bitsandbytes_stacked_params_mapping = {
858
# shard_name, weight_name, index
859
"q_proj": ("qkv_proj", 0),
@@ -1008,10 +1004,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1004
# resampler
1005
".kv_proj.",
1006
]
1011
- # in TP, these weights are partitioned along the column dimension (dim=-1)
1012
- column_parallel_weights_modules = [
1013
- ".down_proj.", ".o_proj.", ".self_attn.out_proj.", ".fc2."
1014
- ]
1007
bitsandbytes_stacked_params_mapping = {
1008
# shard_name, weight_name, index
1009
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/mllama.py
-2
@@ -1062,8 +1062,6 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1062
# so we can't add a dot in front of it.
1063
"multi_modal_projector."
1064
]
1065
- # in TP, these weights are partitioned along the column dimension (dim=-1)
1066
- column_parallel_weights_modules = [".down_proj.", ".o_proj.", ".fc2."]
1065
bitsandbytes_stacked_params_mapping = {
1066
# shard_name, weight_name, index
1067
"q_proj": ("qkv_proj", 0),
vllm/model_executor/models/opt.py
-2
@@ -343,8 +343,6 @@ class OPTForCausalLM(nn.Module, SupportsPP):
343
default_bitsandbytes_target_modules = [
344
".q_proj.", ".k_proj.", ".v_proj.", ".out_proj.", ".fc1.", ".fc2."
345
]
346
- # in TP, these weights are partitioned along the column dimension (dim=-1)
347
- column_parallel_weights_modules = [".out_proj.", ".fc2."]
346
347
def __init__(
348
self,
vllm/model_executor/models/phi.py
-2
@@ -274,8 +274,6 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
274
default_bitsandbytes_target_modules = [
275
".q_proj.", ".k_proj.", ".v_proj.", ".fc1.", ".fc2.", ".dense."
276
]
277
- # in TP, these weights are partitioned along the column dimension (dim=-1)
278
- column_parallel_weights_modules = [".fc2.", ".dense."]
277
278
embedding_modules = {}
279
embedding_padding_modules = []
vllm/model_executor/models/qwen2.py
-3
@@ -395,9 +395,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
395
".v_proj.",
396
".o_proj.",
397
]
398
-
399
- # in TP, these weights are partitioned along the column dimension (dim=-1)
400
- column_parallel_weights_modules = [".down_proj.", ".o_proj."]
398
bitsandbytes_stacked_params_mapping = {
399
# shard_name, weight_name, index
400
"q_proj": ("qkv_proj", 0),