@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fb2716d641
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc]Reduce BNB static variable (#9987)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 5, 2024 at 01:04 UTC fb2716d64117aaa6c36b97b09765aa10a89e2fe5
10 files changed +20 -46
vllm/model_executor/model_loader/loader.py
+20 -20
@@ -28,7 +28,8 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
28 get_tensor_model_parallel_world_size)
29 from vllm.envs import VLLM_USE_MODELSCOPE
30 from vllm.logger import init_logger
31 -from vllm.model_executor.layers.linear import ReplicatedLinear
31 +from vllm.model_executor.layers.linear import (ReplicatedLinear,
32 + RowParallelLinear)
33 from vllm.model_executor.layers.quantization.base_config import (
34 QuantizationConfig)
35 from vllm.model_executor.model_loader.tensorizer import (
@@ -727,6 +728,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
728 def __init__(self, load_config: LoadConfig):
729 super().__init__(load_config)
730
731 + # Save the module names without sharding.
732 + self.unsharded_weights_modules: List[str] = []
733 + # Save the module names that are sharded by column.
734 + self.column_sharded_weights_modules: List[str] = []
735 # we don't need to quantize the whole model, only the target modules
736 # that are specified in the adapter config file. If the adapter config
737 # file is not provided, we will quantize the default modules.
@@ -744,8 +749,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
749 with open(config_file_path, "r") as f:
750 config = json.load(f)
751 self.target_modules = config["target_modules"]
747 - # Save the module names without sharding.
748 - self.unsharded_weights_modules: List[str] = []
752
753 def _get_config_file(self, qlora_adapter: str) -> str:
754 is_local = os.path.isdir(qlora_adapter)
@@ -971,9 +974,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
974 for module in self.unsharded_weights_modules):
975 weight_sub_tensor = weight_tensor
976 # Shard by column
974 - elif any(module in weight_name
975 - for module in self.column_parallel_weights_modules):
976 -
977 + elif any(
978 + weight_name.startswith(module)
979 + for module in self.column_sharded_weights_modules):
980 total_size = weight_tensor.size(-1)
981 start_index = total_size // tp_size * tp_rank
982 end_index = total_size // tp_size * (tp_rank + 1)
@@ -1028,20 +1031,17 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1031 else:
1032 self.target_modules = self.default_target_modules
1033
1031 - if hasattr(model, 'column_parallel_weights_modules'):
1032 - self.column_parallel_weights_modules = \
1033 - model.column_parallel_weights_modules
1034 - else:
1035 - self.column_parallel_weights_modules = []
1036 - # Some modules like `ReplicatedLinear` should not have their weights
1037 - # sharded. The reason for implementing it this way is to avoid new
1038 - # static variable in the model implementation.
1039 - # TODO: Can we reduce the static variables needed for BNB based on
1040 - # model information?
1041 - self.unsharded_weights_modules = [
1042 - name for name, module in model.named_modules()
1043 - if isinstance(module, (ReplicatedLinear, ))
1044 - ]
1034 + for name, module in model.named_modules():
1035 + # Some modules like `ReplicatedLinear` should not have their weights
1036 + # sharded. The reason for implementing it this way is to avoid new
1037 + # static variable in the model implementation.
1038 + if isinstance(module, (ReplicatedLinear, )):
1039 + self.unsharded_weights_modules.append(name)
1040 + # In TP, these weights are partitioned along the column
1041 + # dimension (dim=-1)
1042 + elif isinstance(module, (RowParallelLinear, )):
1043 + self.column_sharded_weights_modules.append(name)
1044 +
1045 self.model_type = type(model).__name__
1046
1047 logger.info("Loading weights with BitsAndBytes quantization. "
vllm/model_executor/models/falcon.py
-2
@@ -401,8 +401,6 @@ class FalconForCausalLM(nn.Module, SupportsPP):
401 ".dense_h_to_4h.",
402 ".dense_4h_to_h.",
403 ]
404 - # in TP, these weights are partitioned along the column dimension (dim=-1)
405 - column_parallel_weights_modules = [".dense_4h_to_h.", ".dense."]
404
405 def __init__(
406 self,
vllm/model_executor/models/gemma.py
-3
@@ -350,7 +350,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
350 "gate_up_proj",
351 "down_proj",
352 ]
353 -
353 # BitandBytes specific attributes
354 default_bitsandbytes_target_modules = [
355 ".gate_proj.",
@@ -361,8 +360,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
360 ".v_proj.",
361 ".o_proj.",
362 ]
364 - # in TP, these weights are partitioned along the column dimension (dim=-1)
365 - column_parallel_weights_modules = [".down_proj.", ".o_proj."]
363 bitsandbytes_stacked_params_mapping = {
364 # shard_name, weight_name, index
365 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/gemma2.py
-2
@@ -390,8 +390,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
390 ".v_proj.",
391 ".o_proj.",
392 ]
393 - # in TP, these weights are partitioned along the column dimension (dim=-1)
394 - column_parallel_weights_modules = [".down_proj.", ".o_proj."]
393 bitsandbytes_stacked_params_mapping = {
394 # shard_name, weight_name, index
395 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/llama.py
-2
@@ -464,8 +464,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
464 ".v_proj.",
465 ".o_proj.",
466 ]
467 - # in TP, these weights are partitioned along the column dimension (dim=-1)
468 - column_parallel_weights_modules = [".down_proj.", ".o_proj."]
467 bitsandbytes_stacked_params_mapping = {
468 # shard_name, weight_name, index
469 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/minicpmv.py
-8
@@ -854,10 +854,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
854 # resampler
855 ".kv_proj.",
856 ]
857 - # in TP, these weights are partitioned along the column dimension (dim=-1)
858 - column_parallel_weights_modules = [
859 - ".down_proj.", ".o_proj.", ".self_attn.out_proj.", ".fc2."
860 - ]
857 bitsandbytes_stacked_params_mapping = {
858 # shard_name, weight_name, index
859 "q_proj": ("qkv_proj", 0),
@@ -1008,10 +1004,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1004 # resampler
1005 ".kv_proj.",
1006 ]
1011 - # in TP, these weights are partitioned along the column dimension (dim=-1)
1012 - column_parallel_weights_modules = [
1013 - ".down_proj.", ".o_proj.", ".self_attn.out_proj.", ".fc2."
1014 - ]
1007 bitsandbytes_stacked_params_mapping = {
1008 # shard_name, weight_name, index
1009 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/mllama.py
-2
@@ -1062,8 +1062,6 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1062 # so we can't add a dot in front of it.
1063 "multi_modal_projector."
1064 ]
1065 - # in TP, these weights are partitioned along the column dimension (dim=-1)
1066 - column_parallel_weights_modules = [".down_proj.", ".o_proj.", ".fc2."]
1065 bitsandbytes_stacked_params_mapping = {
1066 # shard_name, weight_name, index
1067 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/opt.py
-2
@@ -343,8 +343,6 @@ class OPTForCausalLM(nn.Module, SupportsPP):
343 default_bitsandbytes_target_modules = [
344 ".q_proj.", ".k_proj.", ".v_proj.", ".out_proj.", ".fc1.", ".fc2."
345 ]
346 - # in TP, these weights are partitioned along the column dimension (dim=-1)
347 - column_parallel_weights_modules = [".out_proj.", ".fc2."]
346
347 def __init__(
348 self,
vllm/model_executor/models/phi.py
-2
@@ -274,8 +274,6 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
274 default_bitsandbytes_target_modules = [
275 ".q_proj.", ".k_proj.", ".v_proj.", ".fc1.", ".fc2.", ".dense."
276 ]
277 - # in TP, these weights are partitioned along the column dimension (dim=-1)
278 - column_parallel_weights_modules = [".fc2.", ".dense."]
277
278 embedding_modules = {}
279 embedding_padding_modules = []
vllm/model_executor/models/qwen2.py
-3
@@ -395,9 +395,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
395 ".v_proj.",
396 ".o_proj.",
397 ]
398 -
399 - # in TP, these weights are partitioned along the column dimension (dim=-1)
400 - column_parallel_weights_modules = [".down_proj.", ".o_proj."]
398 bitsandbytes_stacked_params_mapping = {
399 # shard_name, weight_name, index
400 "q_proj": ("qkv_proj", 0),