A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[SupportsQuant] Chameleon, Chatglm, Commandr (#15952)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
Kyle Sayers committed
Apr 3, 2025 at 11:25 UTC
82e7e19a6e00bb1c730c29817305aa37ad587210
3 files changed
+17
-8
vllm/model_executor/models/chameleon.py
+7
-2
@@ -38,7 +38,8 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
38
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
39
from vllm.sequence import IntermediateTensors
40
41
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
41
+from .interfaces import (MultiModalEmbeddings, SupportsMultiModal, SupportsPP,
42
+ SupportsQuant)
43
from .utils import (flatten_bn, is_pp_missing_parameter,
44
make_empty_intermediate_tensors_factory, make_layers,
45
maybe_prefix, merge_multimodal_embeddings)
@@ -927,7 +928,11 @@ class ChameleonModel(nn.Module):
928
info=ChameleonProcessingInfo,
929
dummy_inputs=ChameleonDummyInputsBuilder)
930
class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
930
- SupportsPP):
931
+ SupportsPP, SupportsQuant):
932
+ packed_modules_mapping = {
933
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
934
+ "gate_up_proj": ["gate_proj", "up_proj"]
935
+ }
936
937
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
938
super().__init__()
vllm/model_executor/models/chatglm.py
+8
-4
@@ -29,7 +29,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
29
from vllm.sequence import IntermediateTensors
30
from vllm.transformers_utils.configs import ChatGLMConfig
31
32
-from .interfaces import SupportsLoRA, SupportsPP
32
+from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
33
from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
34
make_empty_intermediate_tensors_factory, make_layers,
35
maybe_prefix)
@@ -295,7 +295,11 @@ class GLMTransformer(nn.Module):
295
296
297
@support_torch_compile
298
-class ChatGLMModel(nn.Module):
298
+class ChatGLMModel(nn.Module, SupportsQuant):
299
+ packed_modules_mapping = {
300
+ "linear_proj.merged_proj":
301
+ ["linear_proj.gate_proj", "linear_proj.dense_h_to_4h"]
302
+ }
303
304
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
305
super().__init__()
@@ -395,7 +399,6 @@ class ChatGLMModel(nn.Module):
399
400
401
class ChatGLMBaseModel(nn.Module):
398
-
402
hf_to_vllm_mapper = WeightsMapper(
403
orig_to_new_substr={".word_embeddings": ""}, )
404
@@ -452,7 +455,8 @@ class ChatGLMBaseModel(nn.Module):
455
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
456
457
455
-class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP):
458
+class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
459
+ SupportsQuant):
460
packed_modules_mapping = {
461
"query_key_value": ["query_key_value"],
462
"dense_h_to_4h": ["dense_h_to_4h"]
vllm/model_executor/models/commandr.py
+2
-2
@@ -49,7 +49,7 @@ from vllm.model_executor.utils import set_weight_attrs
49
from vllm.platforms import current_platform
50
from vllm.sequence import IntermediateTensors
51
52
-from .interfaces import SupportsLoRA, SupportsPP
52
+from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
53
from .utils import (extract_layer_index, is_pp_missing_parameter,
54
make_empty_intermediate_tensors_factory, make_layers,
55
maybe_prefix)
@@ -332,7 +332,7 @@ class CohereModel(nn.Module):
332
return hidden_states
333
334
335
-class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
335
+class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
336
packed_modules_mapping = {
337
"qkv_proj": [
338
"q_proj",