A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Misc]Add BNB quantization for PaliGemmaForConditionalGeneration (#12237)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jan 21, 2025 at 15:49 UTC
1f1542afa915e0975d2b63559424403e5e8aae2c
2 files changed
+22
-5
vllm/model_executor/models/paligemma.py
+12
-1
@@ -136,7 +136,18 @@ class PaliGemmaMultiModalProjector(nn.Module):
136
@INPUT_REGISTRY.register_input_processor(input_processor_for_paligemma)
137
class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
138
SupportsPP):
139
-
139
+ packed_modules_mapping = {
140
+ "qkv_proj": [
141
+ "q_proj",
142
+ "k_proj",
143
+ "v_proj",
144
+ ],
145
+ "gate_up_proj": [
146
+ "gate_proj",
147
+ "up_proj",
148
+ ],
149
+ }
150
+
151
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
152
super().__init__()
153
config = vllm_config.model_config.hf_config
vllm/model_executor/models/siglip.py
+10
-4
@@ -344,10 +344,16 @@ class SiglipMLP(nn.Module):
344
345
self.config = config
346
self.activation_fn = get_act_fn(config.hidden_act)
347
-
348
- # For quantization, we require the hidden size to be a multiple of 64
349
- quantizable = (config.hidden_size % 64 == 0
350
- and config.intermediate_size % 64 == 0)
347
+ # Special handling for BNB quantization
348
+ if quant_config and quant_config.get_name() == "bitsandbytes":
349
+ quantizable = True
350
+ else:
351
+ # For other quantization, we require the hidden size to be a
352
+ # multiple of 64
353
+ quantizable = (
354
+ config.hidden_size % 64 == 0
355
+ and config.intermediate_size % 64 == 0
356
+ )
357
self.fc1 = ColumnParallelLinear(
358
config.hidden_size,
359
config.intermediate_size,