@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4c05edb33a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Add TP and BNB quantization support to LlavaMultiModalProjector (#10834)

Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Isotr0py committed Dec 3, 2024 at 07:06 UTC 4c05edb33ae4ae279421ddf981816d070e8ec37a
2 files changed +34 -15
vllm/model_executor/model_loader/loader.py
+11 -3
@@ -1120,7 +1120,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1120 model_config.revision,
1121 pre_quant, load_8bit))
1122
1123 - model.load_weights(qweight_iterator)
1123 + weights_to_load = {name for name, _ in model.named_parameters()}
1124 + loaded_weights = model.load_weights(qweight_iterator)
1125 + # Some models may have weights loading tracker unimplemented.
1126 + if loaded_weights is not None:
1127 + weights_not_loaded = weights_to_load - loaded_weights
1128 + if weights_not_loaded:
1129 + raise ValueError("Following weights were not initialized from "
1130 + f"checkpoint: {weights_not_loaded}")
1131
1132 torch.cuda.empty_cache()
1133
@@ -1152,9 +1159,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1159 shard_name, weight_name)
1160 break
1161
1162 + # Models like Clip/Siglip may skip some layers in initialization,
1163 + # causing unused quant_param_name in state_dict.
1164 if quant_param_name not in param_dict:
1156 - raise ValueError(
1157 - f"Parameter {quant_param_name} not found in the model.")
1165 + continue
1166
1167 if quant_param_name not in stacked_quant_state_dict:
1168 stacked_quant_state_dict[quant_param_name] = {}
vllm/model_executor/models/llava.py
+23 -12
@@ -13,6 +13,8 @@ from vllm.config import VllmConfig
13 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14 InputContext)
15 from vllm.model_executor.layers.activation import get_act_fn
16 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 + RowParallelLinear)
18 from vllm.model_executor.layers.quantization import QuantizationConfig
19 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -59,25 +61,32 @@ class LlavaImageEmbeddingInputs(TypedDict):
61 LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageEmbeddingInputs]
62
63
62 -# TODO(xwjiang): Run benchmark and decide if TP.
64 class LlavaMultiModalProjector(nn.Module):
65
65 - def __init__(self, vision_hidden_size: int, text_hidden_size: int,
66 - projector_hidden_act: str):
66 + def __init__(self,
67 + vision_hidden_size: int,
68 + text_hidden_size: int,
69 + projector_hidden_act: str,
70 + quant_config: Optional[QuantizationConfig] = None,
71 + prefix: str = ""):
72 super().__init__()
73
69 - self.linear_1 = nn.Linear(vision_hidden_size,
70 - text_hidden_size,
71 - bias=True)
74 + self.linear_1 = ColumnParallelLinear(vision_hidden_size,
75 + text_hidden_size,
76 + bias=True,
77 + quant_config=quant_config,
78 + prefix=f"{prefix}.linear_1")
79 self.act = get_act_fn(projector_hidden_act)
73 - self.linear_2 = nn.Linear(text_hidden_size,
74 - text_hidden_size,
75 - bias=True)
80 + self.linear_2 = RowParallelLinear(text_hidden_size,
81 + text_hidden_size,
82 + bias=True,
83 + quant_config=quant_config,
84 + prefix=f"{prefix}.linear_2")
85
86 def forward(self, image_features: torch.Tensor) -> torch.Tensor:
78 - hidden_states = self.linear_1(image_features)
87 + hidden_states, _ = self.linear_1(image_features)
88 hidden_states = self.act(hidden_states)
80 - hidden_states = self.linear_2(hidden_states)
89 + hidden_states, _ = self.linear_2(hidden_states)
90 return hidden_states
91
92
@@ -325,7 +334,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
334 self.multi_modal_projector = LlavaMultiModalProjector(
335 vision_hidden_size=config.vision_config.hidden_size,
336 text_hidden_size=config.text_config.hidden_size,
328 - projector_hidden_act=config.projector_hidden_act)
337 + projector_hidden_act=config.projector_hidden_act,
338 + quant_config=quant_config,
339 + prefix=maybe_prefix(prefix, "multi_modal_projector"))
340
341 self.language_model = init_vllm_registered_model(
342 vllm_config=vllm_config,