A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Model][Quant] Fix GLM, Fix fused module mappings for quantization (#12634)
Signed-off-by: mgoin <michael@neuralmagic.com> Signed-off-by: Kyle Sayers <kylesayrs@gmail.com> Co-authored-by: mgoin <michael@neuralmagic.com>
Kyle Sayers committed
Feb 5, 2025 at 00:32 UTC
7ff7a638b66fdeca71257f245369bd7337e55d32
12 files changed
+195
-151
vllm/model_executor/layers/quantization/base_config.py
+2
-1
@@ -2,7 +2,7 @@
2
3
import inspect
4
from abc import ABC, abstractmethod
5
-from typing import Any, Dict, List, Optional, Type
5
+from typing import Any, Dict, List, Mapping, Optional, Type
6
7
import torch
8
from torch import nn
@@ -59,6 +59,7 @@ def method_has_implemented_embedding(
59
60
class QuantizationConfig(ABC):
61
"""Base class for quantization configs."""
62
+ packed_modules_mapping: Mapping[str, List[str]] = dict()
63
64
@abstractmethod
65
def get_name(self) -> str:
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+17
-20
@@ -83,7 +83,9 @@ class CompressedTensorsConfig(QuantizationConfig):
83
84
# Check if the layer is skipped for quantization.
85
# TODO (@robertgshaw2): support module names
86
- if should_ignore_layer(prefix, ignore=self.ignore):
86
+ if should_ignore_layer(prefix,
87
+ ignore=self.ignore,
88
+ fused_mapping=self.packed_modules_mapping):
89
return UnquantizedLinearMethod()
90
if isinstance(layer, LinearBase):
91
scheme = self.get_scheme(layer=layer, layer_name=prefix)
@@ -379,34 +381,29 @@ class CompressedTensorsConfig(QuantizationConfig):
381
382
# Will be empty for models with only sparsity
383
weight_quant = input_quant = None
382
- sparsity_scheme: Optional[SparsityCompressionConfig] = None
384
if self.target_scheme_map:
385
matched_target = find_matched_target(
386
layer_name=layer_name,
387
module=layer,
387
- targets=self.target_scheme_map.keys())
388
+ targets=self.target_scheme_map.keys(),
389
+ fused_mapping=self.packed_modules_mapping)
390
391
scheme_dict = self.target_scheme_map[matched_target]
392
weight_quant = scheme_dict.get("weights")
393
input_quant = scheme_dict.get("input_activations")
394
393
- if self.sparsity_scheme_map:
394
- is_ignored = False
395
- with suppress(ValueError):
396
- is_ignored = find_matched_target(
397
- layer_name=layer_name,
398
- module=layer,
399
- targets=self.sparsity_ignore_list)
400
-
401
- # if the layer is in the sparsity ignore list,
402
- # we should not apply any sparsity scheme
403
-
404
- if not is_ignored:
405
- matched_target = find_matched_target(
406
- layer_name=layer_name,
407
- module=layer,
408
- targets=self.sparsity_scheme_map.keys())
409
- sparsity_scheme = self.sparsity_scheme_map.get(matched_target)
395
+ # Find the sparsity scheme of the layer
396
+ # assume that fused layers inerhit first component's sparsity scheme
397
+ sparsity_targets = (self.sparsity_scheme_map.keys() -
398
+ set(self.sparsity_ignore_list))
399
+ sparsity_scheme: Optional[SparsityCompressionConfig] = None
400
+ with suppress(ValueError):
401
+ matched_target = find_matched_target(
402
+ layer_name=layer_name,
403
+ module=layer,
404
+ targets=sparsity_targets,
405
+ fused_mapping=self.packed_modules_mapping)
406
+ sparsity_scheme = self.sparsity_scheme_map[matched_target]
407
408
if self.supports_cutlass_24(weight_quant=weight_quant,
409
input_quant=input_quant,
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+55
-85
@@ -1,14 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import re
4
-from typing import Iterable, Optional
4
+from types import MappingProxyType
5
+from typing import Iterable, List, Mapping, Optional
6
7
from compressed_tensors import CompressionFormat
8
from torch.nn import Module
9
9
-from vllm.model_executor.layers.quantization.utils.quant_utils import (
10
- FUSED_LAYER_NAME_MAPPING)
11
-
10
11
def is_activation_quantization_format(format: str) -> bool:
12
_ACTIVATION_QUANTIZATION_FORMATS = [
@@ -19,8 +17,11 @@ def is_activation_quantization_format(format: str) -> bool:
17
return format in _ACTIVATION_QUANTIZATION_FORMATS
18
19
22
-def should_ignore_layer(layer_name: Optional[str],
23
- ignore: Iterable[str]) -> bool:
20
+def should_ignore_layer(
21
+ layer_name: Optional[str],
22
+ ignore: Iterable[str] = tuple(),
23
+ fused_mapping: Mapping[str, List[str]] = MappingProxyType({})
24
+) -> bool:
25
if layer_name is None:
26
return False
27
@@ -32,8 +33,8 @@ def should_ignore_layer(layer_name: Optional[str],
33
# in the safetensors checkpoint. So, we convert the name
34
# from the fused version to unfused + check to make sure that
35
# each shard of the fused layer has the same scheme.
35
- if proj_name in FUSED_LAYER_NAME_MAPPING and layer_name not in ignore:
36
- shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
36
+ if proj_name in fused_mapping and layer_name not in ignore:
37
+ shard_proj_names = fused_mapping[proj_name]
38
39
# Convert fused_name --> [shard_names]
40
shard_names = [
@@ -79,55 +80,12 @@ def check_equal_or_regex_match(layer_name: str,
80
return False
81
82
82
-def _handle_fused_layers(func):
83
- """
84
- Decorator to handle fused layers by mapping vllm fused layer names
85
- to their corresponding unfused layer names for quantization/pruning schemes.
86
- """
87
- # fused_layer_name -> unfused_layer_name
88
- fused_layer_map = {
89
- "qkv_proj": "q_proj",
90
- "gate_up_proj": "up_proj",
91
- }
92
-
93
- def fused_layer_handler(layer_name: Optional[str], module: Module,
94
- targets: Iterable[str]) -> Optional[str]:
95
- """
96
- Wrapper function specifically designed to support the
97
- find_matched_target function.
98
-
99
- It handles cases where the provided layer name corresponds to a
100
- fused layer in vllm, mapping it to its equivalent unfused layer name
101
- based on the predefined fused_layer_map. If the original layer name
102
- raises a ValueError in the wrapped function, this handler
103
- will attempt to resolve the issue by substituting with unfused
104
- layer name.
105
-
106
- :param layer_name: Name of the layer, which may be fused.
107
- :param module: An instance of torch.nn.Module.
108
- :param targets: A list of target names or patterns to match.
109
- :return: The result of the wrapped find_matched_target function with
110
- the resolved layer name.
111
- :raises ValueError: If the layer name cannot be resolved to a
112
- valid target.
113
- """
114
- try:
115
- return func(layer_name, module, targets)
116
- except ValueError:
117
- if layer_name is None:
118
- layer_name = ""
119
- parent_name, fused_proj_name = layer_name.rsplit(".", 1)
120
- unfused_proj_name = fused_layer_map.get(fused_proj_name,
121
- fused_proj_name)
122
- new_layer_name = f"{parent_name}.{unfused_proj_name}"
123
- return func(new_layer_name, module, targets)
124
-
125
- return fused_layer_handler
126
-
127
-
128
-@_handle_fused_layers
129
-def find_matched_target(layer_name: Optional[str], module: Module,
130
- targets: Iterable[str]) -> str:
83
+def find_matched_target(
84
+ layer_name: Optional[str],
85
+ module: Module,
86
+ targets: Iterable[str],
87
+ fused_mapping: Mapping[str, List[str]] = MappingProxyType({})
88
+) -> str:
89
"""
90
Helper function to look up which "target" in the compressed-tensors
91
config that a layer corresponds to.
@@ -141,19 +99,25 @@ def find_matched_target(layer_name: Optional[str], module: Module,
99
100
First, we try to match the layer_name with a target
101
Second, we try to match the module's name with a target
102
+ Third, we try to map the layer_name to a list of fused module names.
103
+ *All* component module names must match in order for a match to be
104
+ successful. A successful match returns the first component target
105
106
:param layer_name: layer name
107
:param module: torch.nn.Module
108
:param targets: list of targets to match the layer against
109
+ :param fused_mapping: map from fused layer names to its components
110
+ :param fused_strategy: either "all" or "any". If using "all", fused
111
+ layers match if "all" of its components match
112
"""
113
114
if layer_name is None:
115
layer_name = ""
116
153
- matched_target = (_find_first_match(layer_name, targets)
154
- or _find_first_match(module.__class__.__name__, targets,
155
- True)
156
- or _match_fused_layer(layer_name, targets))
117
+ matched_target = (
118
+ _find_first_match(layer_name, targets)
119
+ or _find_first_match(module.__class__.__name__, targets, True)
120
+ or _match_fused_layer(layer_name, targets, fused_mapping))
121
122
if matched_target is None:
123
raise ValueError(
@@ -205,11 +169,19 @@ def _is_equal_or_regex_match(value: str,
169
return False
170
171
208
-def _match_fused_layer(layer_name: str,
209
- target_layers: Iterable[str]) -> Optional[str]:
172
+def _match_fused_layer(
173
+ layer_name: str, target_layers: Iterable[str],
174
+ fused_mapping: Mapping[str, List[str]]) -> Optional[str]:
175
"""
176
Match a fused layer name to its corresponding individual layer in
212
- target_layers.
177
+ target_layers. Returns first value in fused_mapping which matches targets
178
+
179
+ Implements an "all" matching strategy where a fused layer matches iff
180
+ "all" of its components match
181
+
182
+ :param layer_name: layer name
183
+ :param target_layers: list of targets to match the layer against
184
+ :param fused_mapping: map from fused layer names to its components
185
186
Examples:
187
layer_name = "model.layers.0.self_attn.qkv_proj"
@@ -217,27 +189,25 @@ def _match_fused_layer(layer_name: str,
189
"model.layers.0.self_attn.k_proj",
190
"model.layers.0.self_attn.v_proj"]
191
"""
220
- # Split into parent path and layer type
221
- # e.g., "model.layers.0.self_attn" and "qkv_proj"
222
- parent_path = ".".join(layer_name.split(".")[:-1])
223
- layer_type = layer_name.split(".")[-1]
224
-
225
- if layer_type not in FUSED_LAYER_NAME_MAPPING:
192
+ # find layer_name in mapping
193
+ fused = next((key for key in fused_mapping if layer_name.endswith(key)),
194
+ None)
195
+ if fused is None:
196
return None
197
228
- possible_layer_types = FUSED_LAYER_NAME_MAPPING[layer_type]
229
-
230
- # Look for a target layer that:
231
- # 1. Has the same parent path
232
- # 2. Ends with one of the possible individual layer types
233
- for target in target_layers:
234
- is_same_parent = parent_path in target
235
- is_matching_type = any(type_suffix in target
236
- for type_suffix in possible_layer_types)
237
-
238
- if is_same_parent and is_matching_type and all(
239
- (f"{parent_path}.{type_suffix}" in target_layers)
240
- for type_suffix in possible_layer_types):
241
- return target
198
+ # expand path of unfused components
199
+ unfused_paths = [
200
+ layer_name.replace(fused, unfused) for unfused in fused_mapping[fused]
201
+ ]
202
243
- return None
203
+ # for each unfused component, find a match in targets
204
+ unfused_matches: List[Optional[str]] = []
205
+ for unfused in unfused_paths:
206
+ for target in target_layers:
207
+ if _is_equal_or_regex_match(unfused, target):
208
+ unfused_matches.append(target)
209
+ break
210
+ else:
211
+ unfused_matches.append(None)
212
+
213
+ return unfused_matches[0] if all(unfused_matches) else None
vllm/model_executor/layers/quantization/quark/quark.py
+5
-5
@@ -18,8 +18,6 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
18
QuarkScheme, QuarkW8A8Fp8, QuarkW8A8Int8)
19
from vllm.model_executor.layers.quantization.quark.utils import (
20
deep_compare, should_ignore_layer)
21
-from vllm.model_executor.layers.quantization.utils.quant_utils import (
22
- FUSED_LAYER_NAME_MAPPING)
21
from vllm.platforms import current_platform
22
23
__all__ = ["QuarkLinearMethod"]
@@ -58,7 +56,9 @@ class QuarkConfig(QuantizationConfig):
56
57
# Check if the layer is skipped for quantization.
58
exclude_layers = cast(List[str], self.quant_config.get("exclude"))
61
- if should_ignore_layer(prefix, ignore=exclude_layers):
59
+ if should_ignore_layer(prefix,
60
+ ignore=exclude_layers,
61
+ fused_mapping=self.packed_modules_mapping):
62
return UnquantizedLinearMethod()
63
if isinstance(layer, LinearBase):
64
scheme = self.get_scheme(layer=layer, layer_name=prefix)
@@ -201,8 +201,8 @@ class QuarkConfig(QuantizationConfig):
201
module: torch.nn.Module) -> Dict[str, Any]:
202
203
proj_name = layer_name.split(".")[-1]
204
- if proj_name in FUSED_LAYER_NAME_MAPPING:
205
- shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
204
+ if proj_name in self.packed_modules_mapping:
205
+ shard_proj_names = self.packed_modules_mapping[proj_name]
206
207
# Convert fused_name --> [shard_names]
208
shard_names = [
vllm/model_executor/layers/quantization/quark/utils.py
+9
-8
@@ -1,10 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import re
4
-from typing import Any, Iterable, Optional
5
-
6
-from vllm.model_executor.layers.quantization.utils.quant_utils import (
7
- FUSED_LAYER_NAME_MAPPING)
4
+from types import MappingProxyType
5
+from typing import Any, Iterable, List, Mapping, Optional
6
7
8
def deep_compare(dict1: Any, dict2: Any) -> bool:
@@ -20,8 +18,11 @@ def deep_compare(dict1: Any, dict2: Any) -> bool:
18
return dict1 == dict2
19
20
23
-def should_ignore_layer(layer_name: Optional[str],
24
- ignore: Iterable[str]) -> bool:
21
+def should_ignore_layer(
22
+ layer_name: Optional[str],
23
+ ignore: Iterable[str],
24
+ fused_mapping: Mapping[str, List[str]] = MappingProxyType({})
25
+) -> bool:
26
if layer_name is None:
27
return False
28
@@ -33,8 +34,8 @@ def should_ignore_layer(layer_name: Optional[str],
34
# in the safetensors checkpoint. So, we convert the name
35
# from the fused version to unfused + check to make sure that
36
# each shard of the fused layer has the same scheme.
36
- if proj_name in FUSED_LAYER_NAME_MAPPING:
37
- shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
37
+ if proj_name in fused_mapping:
38
+ shard_proj_names = fused_mapping[proj_name]
39
40
# Convert fused_name --> [shard_names]
41
shard_names = [
vllm/model_executor/layers/quantization/utils/quant_utils.py
+14
-12
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""This file is used for /tests and /benchmarks"""
3
-from typing import List, Optional, Tuple
3
+from types import MappingProxyType
4
+from typing import List, Mapping, Optional, Tuple
5
6
import numpy
7
import torch
@@ -12,14 +13,6 @@ from vllm.scalar_type import ScalarType, scalar_types
13
SUPPORTED_GPTQ_QUANT_TYPES = [scalar_types.uint4b8, scalar_types.uint8b128]
14
SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
15
15
-# Note: this is a hack. We should update each model to register the
16
-# stacked params and get it from there instead in a future PR.
17
-# fused_name: List[shard_name]
18
-FUSED_LAYER_NAME_MAPPING = {
19
- "qkv_proj": ["q_proj", "k_proj", "v_proj"],
20
- "gate_up_proj": ["gate_proj", "up_proj"]
21
-}
22
-
16
17
# Normalize the group_shape to the full extent for any dims that are -1
18
def _normalize_quant_group_shape(x: torch.Tensor, group_shape: Tuple[int,
@@ -178,14 +171,23 @@ def unpack_quantized_values_into_int32(w_q: torch.Tensor,
171
return res.permute(inv_perm)
172
173
181
-def is_layer_skipped(prefix: str, ignored_layers: List[str]) -> bool:
174
+def is_layer_skipped(
175
+ prefix: str,
176
+ ignored_layers: List[str],
177
+ fused_mapping: Mapping[str, List[str]] = MappingProxyType({})
178
+) -> bool:
179
# prefix: model.layers.0.self_attn.q_proj
180
# proj_name: q_proj
181
proj_name = prefix.split(".")[-1]
185
- if proj_name in FUSED_LAYER_NAME_MAPPING:
182
+
183
+ # Fused layers like gate_up_proj or qkv_proj will not be fused
184
+ # in the safetensors checkpoint. So, we convert the name
185
+ # from the fused version to unfused + check to make sure that
186
+ # each shard of the fused layer has the same scheme.
187
+ if proj_name in fused_mapping:
188
shard_prefixes = [
189
prefix.replace(proj_name, shard_proj_name)
188
- for shard_proj_name in FUSED_LAYER_NAME_MAPPING[proj_name]
190
+ for shard_proj_name in fused_mapping[proj_name]
191
]
192
193
is_skipped = None
vllm/model_executor/model_loader/loader.py
+4
@@ -43,6 +43,7 @@ from vllm.model_executor.model_loader.tensorizer import (
43
TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
44
serialize_vllm_model, tensorizer_weights_iterator)
45
from vllm.model_executor.model_loader.utils import (ParamMapping,
46
+ configure_quant_config,
47
get_model_architecture,
48
set_default_torch_dtype)
49
from vllm.model_executor.model_loader.weight_utils import (
@@ -113,6 +114,9 @@ def _initialize_model(
114
model_config = vllm_config.model_config
115
model_class, _ = get_model_architecture(model_config)
116
117
+ if vllm_config.quant_config is not None:
118
+ configure_quant_config(vllm_config.quant_config, model_class)
119
+
120
signatures = inspect.signature(model_class.__init__)
121
all_params = [param.name for param in signatures.parameters.values()]
122
if "vllm_config" in all_params and "prefix" in all_params:
vllm/model_executor/model_loader/utils.py
+22
@@ -11,6 +11,8 @@ from transformers.dynamic_module_utils import get_class_from_dynamic_module
11
12
from vllm.config import ModelConfig, ModelImpl
13
from vllm.logger import init_logger
14
+from vllm.model_executor.layers.quantization.base_config import (
15
+ QuantizationConfig)
16
from vllm.model_executor.models import ModelRegistry
17
from vllm.model_executor.models.adapters import (as_classification_model,
18
as_embedding_model,
@@ -138,3 +140,23 @@ class ParamMapping:
140
if module_name.endswith(key):
141
return key, value
142
return None
143
+
144
+
145
+def configure_quant_config(quant_config: QuantizationConfig,
146
+ model_class: Type[nn.Module]):
147
+ """
148
+ Pass packed_modules_mapping by reference to quant_config so that
149
+ quant_config can properly match fused modules
150
+
151
+ Note that model attributes are passed by reference to quant_config,
152
+ enabling them to be updated by model_class.__new__ (ex. chatglm, qwen)
153
+ """
154
+ packed_mapping = getattr(model_class, "packed_modules_mapping", None)
155
+ if packed_mapping is not None:
156
+ # pass packed_modules_mapping by reference to quant_config
157
+ quant_config.packed_modules_mapping = packed_mapping
158
+ else:
159
+ logger.warning(
160
+ "The model class %s has not defined `packed_modules_mapping`, "
161
+ "this may lead to incorrect mapping of quantized or ignored "
162
+ "modules", model_class.__name__)
vllm/model_executor/models/chatglm.py
+21
-5
@@ -265,12 +265,14 @@ class GLMAttention(nn.Module):
265
self.total_num_kv_heads,
266
bias=config.add_bias_linear or config.add_qkv_bias,
267
quant_config=quant_config,
268
+ prefix=f"{prefix}.query_key_value",
269
)
270
self.dense = RowParallelLinear(
271
self.total_num_heads * self.head_dim,
272
config.hidden_size,
273
bias=config.add_bias_linear,
274
quant_config=quant_config,
275
+ prefix=f"{prefix}.dense",
276
)
277
278
# https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
@@ -327,6 +329,7 @@ class GLMMLP(nn.Module):
329
self,
330
config: ChatGLMConfig,
331
quant_config: Optional[QuantizationConfig] = None,
332
+ prefix: str = "",
333
):
334
super().__init__()
335
@@ -338,6 +341,7 @@ class GLMMLP(nn.Module):
341
[config.ffn_hidden_size] * 2,
342
bias=config.add_bias_linear,
343
quant_config=quant_config,
344
+ prefix=f"{prefix}.dense_h_to_4h",
345
)
346
347
self.activation_func = SiluAndMul()
@@ -348,6 +352,7 @@ class GLMMLP(nn.Module):
352
config.hidden_size,
353
bias=config.add_bias_linear,
354
quant_config=quant_config,
355
+ prefix=f"{prefix}.dense_4h_to_h",
356
)
357
358
def forward(self, hidden_states):
@@ -396,7 +401,7 @@ class GLMBlock(nn.Module):
401
config.hidden_size, eps=config.layernorm_epsilon)
402
403
# MLP
399
- self.mlp = GLMMLP(config, quant_config)
404
+ self.mlp = GLMMLP(config, quant_config, prefix=f"{prefix}.mlp")
405
406
def forward(
407
self,
@@ -507,7 +512,8 @@ class ChatGLMModel(nn.Module):
512
513
self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
514
config.hidden_size,
510
- quant_config=quant_config)
515
+ quant_config=quant_config,
516
+ prefix=f"{prefix}.embedding")
517
518
self.num_layers = config.num_layers
519
self.multi_query_group_num = config.multi_query_group_num
@@ -766,6 +772,7 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
772
SupportsMultiModal):
773
# Ensure that the LoRA support check passes when the class is not
774
# initialized, but set all these attributes to empty.
775
+ # These will be updated when an instance class is selected
776
packed_modules_mapping = {}
777
supported_lora_modules = []
778
embedding_modules = {}
@@ -777,9 +784,18 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
784
prefix: str = "",
785
) -> None:
786
config = vllm_config.model_config.hf_config
787
+
788
# Initialize VL
781
- if hasattr(config, "vision_config"):
782
- return ChatGLMV(vllm_config=vllm_config, prefix=prefix)
789
+ if hasattr(config, "vision_config"): # noqa: SIM108
790
+ instance_cls = ChatGLMV
791
# Initialize LLM
792
else:
785
- return ChatGLM(vllm_config=vllm_config, prefix=prefix)
\ No newline at end of file
793
+ instance_cls = ChatGLM
794
+
795
+ # quant_config references base class members,
796
+ # so update values before init is called
797
+ cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
798
+ cls.supported_lora_modules += instance_cls.supported_lora_modules
799
+ cls.embedding_modules.update(instance_cls.embedding_modules)
800
+ cls.embedding_padding_modules += instance_cls.embedding_padding_modules
801
+ return instance_cls(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/glm4_vision_encoder.py
+22
-9
@@ -74,11 +74,13 @@ class Attention(nn.Module):
74
self.head_dim,
75
config.num_heads,
76
quant_config=quant_config,
77
+ prefix=f"{prefix}.query_key_value",
78
)
79
self.dense = RowParallelLinear(
80
config.hidden_size,
81
config.hidden_size,
82
quant_config=quant_config,
83
+ prefix=f"{prefix}.dense",
84
)
85
86
self.attn = MultiHeadAttention(self.num_heads_per_rank, self.head_dim,
@@ -101,6 +103,7 @@ class MLP(nn.Module):
103
self,
104
config,
105
quant_config: Optional[QuantizationConfig] = None,
106
+ prefix: str = '',
107
):
108
super().__init__()
109
self.config = config
@@ -109,11 +112,13 @@ class MLP(nn.Module):
112
config.hidden_size,
113
config.intermediate_size,
114
quant_config=quant_config,
115
+ prefix=f"{prefix}.fc1",
116
)
117
self.fc2 = RowParallelLinear(
118
config.intermediate_size,
119
config.hidden_size,
120
quant_config=quant_config,
121
+ prefix=f"{prefix}.fc2",
122
)
123
124
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -137,7 +142,9 @@ class TransformerLayer(nn.Module):
142
self.attention = Attention(config,
143
quant_config=quant_config,
144
prefix=f"{prefix}.attention")
140
- self.mlp = MLP(config, quant_config=quant_config)
145
+ self.mlp = MLP(config,
146
+ quant_config=quant_config,
147
+ prefix=f"{prefix}.mlp")
148
self.post_attention_layernorm = LayerNorm(config.hidden_size,
149
eps=config.layer_norm_eps)
150
@@ -164,7 +171,7 @@ class Transformer(nn.Module):
171
self.layers = nn.ModuleList([
172
TransformerLayer(config,
173
quant_config=quant_config,
167
- prefix=f"{prefix}.layer.{layer_idx}")
174
+ prefix=f"{prefix}.layers.{layer_idx}")
175
for layer_idx in range(config.num_hidden_layers)
176
])
177
@@ -181,6 +188,7 @@ class GLU(nn.Module):
188
config,
189
in_features,
190
quant_config: Optional[QuantizationConfig] = None,
191
+ prefix: str = '',
192
):
193
"""
194
The original implementation is the same as:
@@ -222,7 +230,8 @@ class GLU(nn.Module):
230
self.linear_proj = ReplicatedLinear(in_features,
231
config.hidden_size,
232
bias=False,
225
- quant_config=quant_config)
233
+ quant_config=quant_config,
234
+ prefix=f"{prefix}.linear_proj")
235
self.norm1 = nn.LayerNorm(config.hidden_size)
236
self.act1 = nn.GELU()
237
self.act2 = SiluAndMul()
@@ -230,12 +239,15 @@ class GLU(nn.Module):
239
self.merged_proj = MergedColumnParallelLinear(
240
config.hidden_size, [config.ffn_hidden_size] * 2,
241
bias=False,
233
- quant_config=quant_config)
242
+ quant_config=quant_config,
243
+ prefix=f"{prefix}.merged_proj")
244
235
- self.dense_4h_to_h = RowParallelLinear(config.ffn_hidden_size,
236
- config.hidden_size,
237
- bias=False,
238
- quant_config=quant_config)
245
+ self.dense_4h_to_h = RowParallelLinear(
246
+ config.ffn_hidden_size,
247
+ config.hidden_size,
248
+ bias=False,
249
+ quant_config=quant_config,
250
+ prefix=f"{prefix}.dense_4h_to_h")
251
252
def forward(self, x):
253
x, _ = self.linear_proj(x)
@@ -262,7 +274,8 @@ class EVA2CLIPModel(nn.Module):
274
prefix=f"{prefix}.transformer")
275
self.linear_proj = GLU(config,
276
in_features=config.hidden_size,
265
- quant_config=quant_config)
277
+ quant_config=quant_config,
278
+ prefix=f"{prefix}.linear_proj")
279
self.conv = nn.Conv2d(in_channels=vision_config.hidden_size,
280
out_channels=config.hidden_size,
281
kernel_size=2,
vllm/model_executor/models/minicpmv.py
+11
-3
@@ -1473,6 +1473,7 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
1473
"""
1474
# Ensure that the LoRA support check passes when the class is not
1475
# initialized, but set all these attributes to empty.
1476
+ # These will be updated when an instance class is selected
1477
packed_modules_mapping = {}
1478
supported_lora_modules = []
1479
embedding_modules = {}
@@ -1489,8 +1490,15 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
1490
version = str(config.version).split(".")
1491
version = tuple([int(x) for x in version])
1492
# Dispatch class based on version
1492
- instance_class = _SUPPORT_VERSION.get(version)
1493
- if instance_class is None:
1493
+ instance_cls = _SUPPORT_VERSION.get(version)
1494
+ if instance_cls is None:
1495
raise ValueError(
1496
"Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
1496
- return instance_class(vllm_config=vllm_config, prefix=prefix)
1497
+
1498
+ # quant_config references base class members,
1499
+ # so update values before init is called
1500
+ cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
1501
+ cls.supported_lora_modules += instance_cls.supported_lora_modules
1502
+ cls.embedding_modules.update(instance_cls.embedding_modules)
1503
+ cls.embedding_padding_modules += instance_cls.embedding_padding_modules
1504
+ return instance_cls(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/qwen.py
+13
-3
@@ -1135,6 +1135,7 @@ class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1135
"""
1136
# Ensure that the LoRA support check passes when the class is not
1137
# initialized, but set all these attributes to empty.
1138
+ # These will be updated when an instance class is selected
1139
packed_modules_mapping = {}
1140
supported_lora_modules = []
1141
embedding_modules = {}
@@ -1146,9 +1147,18 @@ class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1147
prefix: str = "",
1148
) -> QWenBaseModel:
1149
config = vllm_config.model_config.hf_config
1150
+
1151
# Initialize VL
1150
- if hasattr(config, "visual"):
1151
- return QWenVL(vllm_config=vllm_config, prefix=prefix)
1152
+ if hasattr(config, "visual"): # noqa: SIM108
1153
+ instance_cls = QWenVL
1154
# Initialize LLM
1155
else:
1154
- return QWenLLM(vllm_config=vllm_config, prefix=prefix)
1156
+ instance_cls = QWenLLM
1157
+
1158
+ # quant_config references base class members,
1159
+ # so update values before init is called
1160
+ cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
1161
+ cls.supported_lora_modules += instance_cls.supported_lora_modules
1162
+ cls.embedding_modules.update(instance_cls.embedding_modules)
1163
+ cls.embedding_padding_modules += instance_cls.embedding_padding_modules
1164
+ return instance_cls(vllm_config=vllm_config, prefix=prefix)