@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5be4e52b65
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model][LoRA]LoRA support added for glm-4v (#10418)

Signed-off-by: B-201 <Joy25810@foxmail.com>

B-201 committed Nov 18, 2024 at 20:57 UTC 5be4e52b6522113f7276e60b32cb5c1f912de6fd
1 file changed +79 -19
vllm/model_executor/models/chatglm.py
+79 -19
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
30 ParallelLMHead, VocabParallelEmbedding)
31 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
32 from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33 +from vllm.model_executor.models.module_mapping import MultiModelKeys
34 from vllm.model_executor.sampling_metadata import SamplingMetadata
35 from vllm.multimodal import MULTIMODAL_REGISTRY
36 from vllm.multimodal.inputs import MultiModalData, MultiModalKwargs
@@ -574,25 +575,8 @@ class ChatGLMModel(nn.Module):
575 return hidden_states
576
577
577 -@MULTIMODAL_REGISTRY.register_image_input_mapper(mm_input_mapper_for_glmv)
578 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
579 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
580 -@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
581 -class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
582 - SupportsMultiModal):
583 - packed_modules_mapping = {
584 - "query_key_value": ["query_key_value"],
585 - "dense_h_to_4h": ["dense_h_to_4h"]
586 - }
587 - # LoRA specific attributes
588 - supported_lora_modules = [
589 - "query_key_value",
590 - "dense",
591 - "dense_h_to_4h",
592 - "dense_4h_to_h",
593 - ]
594 - embedding_modules = {}
595 - embedding_padding_modules = []
578 +class ChatGLMBaseModel(nn.Module, SupportsLoRA, SupportsPP,
579 + SupportsMultiModal):
580
581 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
582 super().__init__()
@@ -692,3 +676,79 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
676 weight_loader(param, combined_weight)
677 loaded_params.add(combined_name)
678 return loaded_params
679 +
680 +
681 +class ChatGLM(ChatGLMBaseModel):
682 + packed_modules_mapping = {
683 + "query_key_value": ["query_key_value"],
684 + "dense_h_to_4h": ["dense_h_to_4h"]
685 + }
686 + # LoRA specific attributes
687 + supported_lora_modules = [
688 + "query_key_value",
689 + "dense",
690 + "dense_h_to_4h",
691 + "dense_4h_to_h",
692 + ]
693 +
694 + embedding_modules = {}
695 + embedding_padding_modules = []
696 +
697 +
698 +class ChatGLMV(ChatGLMBaseModel):
699 + packed_modules_mapping = {
700 + "query_key_value": ["query_key_value"],
701 + "dense_h_to_4h": ["dense_h_to_4h"],
702 + "merged_proj": ["gate_proj", "dense_h_to_4h"]
703 + }
704 + # LoRA specific attributes
705 + supported_lora_modules = [
706 + "query_key_value",
707 + "dense",
708 + "dense_h_to_4h",
709 + "dense_4h_to_h",
710 + # vision
711 + "fc1",
712 + "fc2",
713 + "merged_proj",
714 + "linear_proj"
715 + ]
716 +
717 + embedding_modules = {}
718 + embedding_padding_modules = []
719 +
720 + def get_mm_mapping(self) -> MultiModelKeys:
721 + """
722 + Get the module prefix in multimodal models
723 + """
724 + return MultiModelKeys.from_string_field(
725 + language_model="transformer.encoder",
726 + connector="transformer.vision.linear_proj",
727 + tower_model="transformer.vision.transformer")
728 +
729 +
730 +@MULTIMODAL_REGISTRY.register_image_input_mapper(mm_input_mapper_for_glmv)
731 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
732 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
733 +@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
734 +class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
735 + SupportsMultiModal):
736 + # Ensure that the LoRA support check passes when the class is not
737 + # initialized, but set all these attributes to empty.
738 + packed_modules_mapping = {}
739 + supported_lora_modules = []
740 + embedding_modules = {}
741 + embedding_padding_modules = []
742 +
743 + def __new__(
744 + cls,
745 + vllm_config: VllmConfig,
746 + prefix: str = "",
747 + ) -> None:
748 + config = vllm_config.model_config.hf_config
749 + # Initialize VL
750 + if hasattr(config, "visual"):
751 + return ChatGLM(vllm_config=vllm_config, prefix=prefix)
752 + # Initialize LLM
753 + else:
754 + return ChatGLMV(vllm_config=vllm_config, prefix=prefix)