@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b18c9bbaba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Add BNB support to Llava and Pixtral-HF (#10795)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Dec 2, 2024 at 09:31 UTC b18c9bbaba6e1c6dfb92fe52e5a6cb22dd6bfa81
1 file changed +9
vllm/model_executor/models/llava.py
+9
@@ -287,6 +287,15 @@ def init_vision_tower_for_llava(
287 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
288 @INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
289 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
290 + # BitandBytes specific attributes
291 + bitsandbytes_stacked_params_mapping = {
292 + # shard_name, weight_name, index
293 + "q_proj": ("qkv_proj", 0),
294 + "k_proj": ("qkv_proj", 1),
295 + "v_proj": ("qkv_proj", 2),
296 + "gate_proj": ("gate_up_proj", 0),
297 + "up_proj": ("gate_up_proj", 1),
298 + }
299
300 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
301 super().__init__()