A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Add BNB support to Llava and Pixtral-HF (#10795)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Dec 2, 2024 at 09:31 UTC
b18c9bbaba6e1c6dfb92fe52e5a6cb22dd6bfa81
1 file changed
+9
vllm/model_executor/models/llava.py
+9
@@ -287,6 +287,15 @@ def init_vision_tower_for_llava(
287
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
288
@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
289
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
290
+ # BitandBytes specific attributes
291
+ bitsandbytes_stacked_params_mapping = {
292
+ # shard_name, weight_name, index
293
+ "q_proj": ("qkv_proj", 0),
294
+ "k_proj": ("qkv_proj", 1),
295
+ "v_proj": ("qkv_proj", 2),
296
+ "gate_proj": ("gate_up_proj", 0),
297
+ "up_proj": ("gate_up_proj", 1),
298
+ }
299
300
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
301
super().__init__()