A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Add TP and BNB quantization support to LlavaMultiModalProjector (#10834)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Isotr0py committed
Dec 3, 2024 at 07:06 UTC
4c05edb33ae4ae279421ddf981816d070e8ec37a
2 files changed
+34
-15
vllm/model_executor/model_loader/loader.py
+11
-3
@@ -1120,7 +1120,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1120
model_config.revision,
1121
pre_quant, load_8bit))
1122
1123
- model.load_weights(qweight_iterator)
1123
+ weights_to_load = {name for name, _ in model.named_parameters()}
1124
+ loaded_weights = model.load_weights(qweight_iterator)
1125
+ # Some models may have weights loading tracker unimplemented.
1126
+ if loaded_weights is not None:
1127
+ weights_not_loaded = weights_to_load - loaded_weights
1128
+ if weights_not_loaded:
1129
+ raise ValueError("Following weights were not initialized from "
1130
+ f"checkpoint: {weights_not_loaded}")
1131
1132
torch.cuda.empty_cache()
1133
@@ -1152,9 +1159,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1159
shard_name, weight_name)
1160
break
1161
1162
+ # Models like Clip/Siglip may skip some layers in initialization,
1163
+ # causing unused quant_param_name in state_dict.
1164
if quant_param_name not in param_dict:
1156
- raise ValueError(
1157
- f"Parameter {quant_param_name} not found in the model.")
1165
+ continue
1166
1167
if quant_param_name not in stacked_quant_state_dict:
1168
stacked_quant_state_dict[quant_param_name] = {}
vllm/model_executor/models/llava.py
+23
-12
@@ -13,6 +13,8 @@ from vllm.config import VllmConfig
13
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14
InputContext)
15
from vllm.model_executor.layers.activation import get_act_fn
16
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
+ RowParallelLinear)
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -59,25 +61,32 @@ class LlavaImageEmbeddingInputs(TypedDict):
61
LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageEmbeddingInputs]
62
63
62
-# TODO(xwjiang): Run benchmark and decide if TP.
64
class LlavaMultiModalProjector(nn.Module):
65
65
- def __init__(self, vision_hidden_size: int, text_hidden_size: int,
66
- projector_hidden_act: str):
66
+ def __init__(self,
67
+ vision_hidden_size: int,
68
+ text_hidden_size: int,
69
+ projector_hidden_act: str,
70
+ quant_config: Optional[QuantizationConfig] = None,
71
+ prefix: str = ""):
72
super().__init__()
73
69
- self.linear_1 = nn.Linear(vision_hidden_size,
70
- text_hidden_size,
71
- bias=True)
74
+ self.linear_1 = ColumnParallelLinear(vision_hidden_size,
75
+ text_hidden_size,
76
+ bias=True,
77
+ quant_config=quant_config,
78
+ prefix=f"{prefix}.linear_1")
79
self.act = get_act_fn(projector_hidden_act)
73
- self.linear_2 = nn.Linear(text_hidden_size,
74
- text_hidden_size,
75
- bias=True)
80
+ self.linear_2 = RowParallelLinear(text_hidden_size,
81
+ text_hidden_size,
82
+ bias=True,
83
+ quant_config=quant_config,
84
+ prefix=f"{prefix}.linear_2")
85
86
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
78
- hidden_states = self.linear_1(image_features)
87
+ hidden_states, _ = self.linear_1(image_features)
88
hidden_states = self.act(hidden_states)
80
- hidden_states = self.linear_2(hidden_states)
89
+ hidden_states, _ = self.linear_2(hidden_states)
90
return hidden_states
91
92
@@ -325,7 +334,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
334
self.multi_modal_projector = LlavaMultiModalProjector(
335
vision_hidden_size=config.vision_config.hidden_size,
336
text_hidden_size=config.text_config.hidden_size,
328
- projector_hidden_act=config.projector_hidden_act)
337
+ projector_hidden_act=config.projector_hidden_act,
338
+ quant_config=quant_config,
339
+ prefix=maybe_prefix(prefix, "multi_modal_projector"))
340
341
self.language_model = init_vllm_registered_model(
342
vllm_config=vllm_config,