@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f1542afa9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc]Add BNB quantization for PaliGemmaForConditionalGeneration (#12237)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Jan 21, 2025 at 15:49 UTC 1f1542afa915e0975d2b63559424403e5e8aae2c
2 files changed +22 -5
vllm/model_executor/models/paligemma.py
+12 -1
@@ -136,7 +136,18 @@ class PaliGemmaMultiModalProjector(nn.Module):
136 @INPUT_REGISTRY.register_input_processor(input_processor_for_paligemma)
137 class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
138 SupportsPP):
139 -
139 + packed_modules_mapping = {
140 + "qkv_proj": [
141 + "q_proj",
142 + "k_proj",
143 + "v_proj",
144 + ],
145 + "gate_up_proj": [
146 + "gate_proj",
147 + "up_proj",
148 + ],
149 + }
150 +
151 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
152 super().__init__()
153 config = vllm_config.model_config.hf_config
vllm/model_executor/models/siglip.py
+10 -4
@@ -344,10 +344,16 @@ class SiglipMLP(nn.Module):
344
345 self.config = config
346 self.activation_fn = get_act_fn(config.hidden_act)
347 -
348 - # For quantization, we require the hidden size to be a multiple of 64
349 - quantizable = (config.hidden_size % 64 == 0
350 - and config.intermediate_size % 64 == 0)
347 + # Special handling for BNB quantization
348 + if quant_config and quant_config.get_name() == "bitsandbytes":
349 + quantizable = True
350 + else:
351 + # For other quantization, we require the hidden size to be a
352 + # multiple of 64
353 + quantizable = (
354 + config.hidden_size % 64 == 0
355 + and config.intermediate_size % 64 == 0
356 + )
357 self.fc1 = ColumnParallelLinear(
358 config.hidden_size,
359 config.intermediate_size,