A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model][LoRA]LoRA support added for glm-4v (#10418)
Signed-off-by: B-201 <Joy25810@foxmail.com>
B-201 committed
Nov 18, 2024 at 20:57 UTC
5be4e52b6522113f7276e60b32cb5c1f912de6fd
1 file changed
+79
-19
vllm/model_executor/models/chatglm.py
+79
-19
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
30
ParallelLMHead, VocabParallelEmbedding)
31
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
32
from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33
+from vllm.model_executor.models.module_mapping import MultiModelKeys
34
from vllm.model_executor.sampling_metadata import SamplingMetadata
35
from vllm.multimodal import MULTIMODAL_REGISTRY
36
from vllm.multimodal.inputs import MultiModalData, MultiModalKwargs
@@ -574,25 +575,8 @@ class ChatGLMModel(nn.Module):
575
return hidden_states
576
577
577
-@MULTIMODAL_REGISTRY.register_image_input_mapper(mm_input_mapper_for_glmv)
578
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
579
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
580
-@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
581
-class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
582
- SupportsMultiModal):
583
- packed_modules_mapping = {
584
- "query_key_value": ["query_key_value"],
585
- "dense_h_to_4h": ["dense_h_to_4h"]
586
- }
587
- # LoRA specific attributes
588
- supported_lora_modules = [
589
- "query_key_value",
590
- "dense",
591
- "dense_h_to_4h",
592
- "dense_4h_to_h",
593
- ]
594
- embedding_modules = {}
595
- embedding_padding_modules = []
578
+class ChatGLMBaseModel(nn.Module, SupportsLoRA, SupportsPP,
579
+ SupportsMultiModal):
580
581
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
582
super().__init__()
@@ -692,3 +676,79 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
676
weight_loader(param, combined_weight)
677
loaded_params.add(combined_name)
678
return loaded_params
679
+
680
+
681
+class ChatGLM(ChatGLMBaseModel):
682
+ packed_modules_mapping = {
683
+ "query_key_value": ["query_key_value"],
684
+ "dense_h_to_4h": ["dense_h_to_4h"]
685
+ }
686
+ # LoRA specific attributes
687
+ supported_lora_modules = [
688
+ "query_key_value",
689
+ "dense",
690
+ "dense_h_to_4h",
691
+ "dense_4h_to_h",
692
+ ]
693
+
694
+ embedding_modules = {}
695
+ embedding_padding_modules = []
696
+
697
+
698
+class ChatGLMV(ChatGLMBaseModel):
699
+ packed_modules_mapping = {
700
+ "query_key_value": ["query_key_value"],
701
+ "dense_h_to_4h": ["dense_h_to_4h"],
702
+ "merged_proj": ["gate_proj", "dense_h_to_4h"]
703
+ }
704
+ # LoRA specific attributes
705
+ supported_lora_modules = [
706
+ "query_key_value",
707
+ "dense",
708
+ "dense_h_to_4h",
709
+ "dense_4h_to_h",
710
+ # vision
711
+ "fc1",
712
+ "fc2",
713
+ "merged_proj",
714
+ "linear_proj"
715
+ ]
716
+
717
+ embedding_modules = {}
718
+ embedding_padding_modules = []
719
+
720
+ def get_mm_mapping(self) -> MultiModelKeys:
721
+ """
722
+ Get the module prefix in multimodal models
723
+ """
724
+ return MultiModelKeys.from_string_field(
725
+ language_model="transformer.encoder",
726
+ connector="transformer.vision.linear_proj",
727
+ tower_model="transformer.vision.transformer")
728
+
729
+
730
+@MULTIMODAL_REGISTRY.register_image_input_mapper(mm_input_mapper_for_glmv)
731
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
732
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
733
+@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
734
+class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
735
+ SupportsMultiModal):
736
+ # Ensure that the LoRA support check passes when the class is not
737
+ # initialized, but set all these attributes to empty.
738
+ packed_modules_mapping = {}
739
+ supported_lora_modules = []
740
+ embedding_modules = {}
741
+ embedding_padding_modules = []
742
+
743
+ def __new__(
744
+ cls,
745
+ vllm_config: VllmConfig,
746
+ prefix: str = "",
747
+ ) -> None:
748
+ config = vllm_config.model_config.hf_config
749
+ # Initialize VL
750
+ if hasattr(config, "visual"):
751
+ return ChatGLM(vllm_config=vllm_config, prefix=prefix)
752
+ # Initialize LLM
753
+ else:
754
+ return ChatGLMV(vllm_config=vllm_config, prefix=prefix)