@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7ff7a638b6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Model][Quant] Fix GLM, Fix fused module mappings for quantization (#12634)

Signed-off-by: mgoin <michael@neuralmagic.com> Signed-off-by: Kyle Sayers <kylesayrs@gmail.com> Co-authored-by: mgoin <michael@neuralmagic.com>

Kyle Sayers committed Feb 5, 2025 at 00:32 UTC 7ff7a638b66fdeca71257f245369bd7337e55d32
12 files changed +195 -151
vllm/model_executor/layers/quantization/base_config.py
+2 -1
@@ -2,7 +2,7 @@
2
3 import inspect
4 from abc import ABC, abstractmethod
5 -from typing import Any, Dict, List, Optional, Type
5 +from typing import Any, Dict, List, Mapping, Optional, Type
6
7 import torch
8 from torch import nn
@@ -59,6 +59,7 @@ def method_has_implemented_embedding(
59
60 class QuantizationConfig(ABC):
61 """Base class for quantization configs."""
62 + packed_modules_mapping: Mapping[str, List[str]] = dict()
63
64 @abstractmethod
65 def get_name(self) -> str:
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+17 -20
@@ -83,7 +83,9 @@ class CompressedTensorsConfig(QuantizationConfig):
83
84 # Check if the layer is skipped for quantization.
85 # TODO (@robertgshaw2): support module names
86 - if should_ignore_layer(prefix, ignore=self.ignore):
86 + if should_ignore_layer(prefix,
87 + ignore=self.ignore,
88 + fused_mapping=self.packed_modules_mapping):
89 return UnquantizedLinearMethod()
90 if isinstance(layer, LinearBase):
91 scheme = self.get_scheme(layer=layer, layer_name=prefix)
@@ -379,34 +381,29 @@ class CompressedTensorsConfig(QuantizationConfig):
381
382 # Will be empty for models with only sparsity
383 weight_quant = input_quant = None
382 - sparsity_scheme: Optional[SparsityCompressionConfig] = None
384 if self.target_scheme_map:
385 matched_target = find_matched_target(
386 layer_name=layer_name,
387 module=layer,
387 - targets=self.target_scheme_map.keys())
388 + targets=self.target_scheme_map.keys(),
389 + fused_mapping=self.packed_modules_mapping)
390
391 scheme_dict = self.target_scheme_map[matched_target]
392 weight_quant = scheme_dict.get("weights")
393 input_quant = scheme_dict.get("input_activations")
394
393 - if self.sparsity_scheme_map:
394 - is_ignored = False
395 - with suppress(ValueError):
396 - is_ignored = find_matched_target(
397 - layer_name=layer_name,
398 - module=layer,
399 - targets=self.sparsity_ignore_list)
400 -
401 - # if the layer is in the sparsity ignore list,
402 - # we should not apply any sparsity scheme
403 -
404 - if not is_ignored:
405 - matched_target = find_matched_target(
406 - layer_name=layer_name,
407 - module=layer,
408 - targets=self.sparsity_scheme_map.keys())
409 - sparsity_scheme = self.sparsity_scheme_map.get(matched_target)
395 + # Find the sparsity scheme of the layer
396 + # assume that fused layers inerhit first component's sparsity scheme
397 + sparsity_targets = (self.sparsity_scheme_map.keys() -
398 + set(self.sparsity_ignore_list))
399 + sparsity_scheme: Optional[SparsityCompressionConfig] = None
400 + with suppress(ValueError):
401 + matched_target = find_matched_target(
402 + layer_name=layer_name,
403 + module=layer,
404 + targets=sparsity_targets,
405 + fused_mapping=self.packed_modules_mapping)
406 + sparsity_scheme = self.sparsity_scheme_map[matched_target]
407
408 if self.supports_cutlass_24(weight_quant=weight_quant,
409 input_quant=input_quant,
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+55 -85
@@ -1,14 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import re
4 -from typing import Iterable, Optional
4 +from types import MappingProxyType
5 +from typing import Iterable, List, Mapping, Optional
6
7 from compressed_tensors import CompressionFormat
8 from torch.nn import Module
9
9 -from vllm.model_executor.layers.quantization.utils.quant_utils import (
10 - FUSED_LAYER_NAME_MAPPING)
11 -
10
11 def is_activation_quantization_format(format: str) -> bool:
12 _ACTIVATION_QUANTIZATION_FORMATS = [
@@ -19,8 +17,11 @@ def is_activation_quantization_format(format: str) -> bool:
17 return format in _ACTIVATION_QUANTIZATION_FORMATS
18
19
22 -def should_ignore_layer(layer_name: Optional[str],
23 - ignore: Iterable[str]) -> bool:
20 +def should_ignore_layer(
21 + layer_name: Optional[str],
22 + ignore: Iterable[str] = tuple(),
23 + fused_mapping: Mapping[str, List[str]] = MappingProxyType({})
24 +) -> bool:
25 if layer_name is None:
26 return False
27
@@ -32,8 +33,8 @@ def should_ignore_layer(layer_name: Optional[str],
33 # in the safetensors checkpoint. So, we convert the name
34 # from the fused version to unfused + check to make sure that
35 # each shard of the fused layer has the same scheme.
35 - if proj_name in FUSED_LAYER_NAME_MAPPING and layer_name not in ignore:
36 - shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
36 + if proj_name in fused_mapping and layer_name not in ignore:
37 + shard_proj_names = fused_mapping[proj_name]
38
39 # Convert fused_name --> [shard_names]
40 shard_names = [
@@ -79,55 +80,12 @@ def check_equal_or_regex_match(layer_name: str,
80 return False
81
82
82 -def _handle_fused_layers(func):
83 - """
84 - Decorator to handle fused layers by mapping vllm fused layer names
85 - to their corresponding unfused layer names for quantization/pruning schemes.
86 - """
87 - # fused_layer_name -> unfused_layer_name
88 - fused_layer_map = {
89 - "qkv_proj": "q_proj",
90 - "gate_up_proj": "up_proj",
91 - }
92 -
93 - def fused_layer_handler(layer_name: Optional[str], module: Module,
94 - targets: Iterable[str]) -> Optional[str]:
95 - """
96 - Wrapper function specifically designed to support the
97 - find_matched_target function.
98 -
99 - It handles cases where the provided layer name corresponds to a
100 - fused layer in vllm, mapping it to its equivalent unfused layer name
101 - based on the predefined fused_layer_map. If the original layer name
102 - raises a ValueError in the wrapped function, this handler
103 - will attempt to resolve the issue by substituting with unfused
104 - layer name.
105 -
106 - :param layer_name: Name of the layer, which may be fused.
107 - :param module: An instance of torch.nn.Module.
108 - :param targets: A list of target names or patterns to match.
109 - :return: The result of the wrapped find_matched_target function with
110 - the resolved layer name.
111 - :raises ValueError: If the layer name cannot be resolved to a
112 - valid target.
113 - """
114 - try:
115 - return func(layer_name, module, targets)
116 - except ValueError:
117 - if layer_name is None:
118 - layer_name = ""
119 - parent_name, fused_proj_name = layer_name.rsplit(".", 1)
120 - unfused_proj_name = fused_layer_map.get(fused_proj_name,
121 - fused_proj_name)
122 - new_layer_name = f"{parent_name}.{unfused_proj_name}"
123 - return func(new_layer_name, module, targets)
124 -
125 - return fused_layer_handler
126 -
127 -
128 -@_handle_fused_layers
129 -def find_matched_target(layer_name: Optional[str], module: Module,
130 - targets: Iterable[str]) -> str:
83 +def find_matched_target(
84 + layer_name: Optional[str],
85 + module: Module,
86 + targets: Iterable[str],
87 + fused_mapping: Mapping[str, List[str]] = MappingProxyType({})
88 +) -> str:
89 """
90 Helper function to look up which "target" in the compressed-tensors
91 config that a layer corresponds to.
@@ -141,19 +99,25 @@ def find_matched_target(layer_name: Optional[str], module: Module,
99
100 First, we try to match the layer_name with a target
101 Second, we try to match the module's name with a target
102 + Third, we try to map the layer_name to a list of fused module names.
103 + *All* component module names must match in order for a match to be
104 + successful. A successful match returns the first component target
105
106 :param layer_name: layer name
107 :param module: torch.nn.Module
108 :param targets: list of targets to match the layer against
109 + :param fused_mapping: map from fused layer names to its components
110 + :param fused_strategy: either "all" or "any". If using "all", fused
111 + layers match if "all" of its components match
112 """
113
114 if layer_name is None:
115 layer_name = ""
116
153 - matched_target = (_find_first_match(layer_name, targets)
154 - or _find_first_match(module.__class__.__name__, targets,
155 - True)
156 - or _match_fused_layer(layer_name, targets))
117 + matched_target = (
118 + _find_first_match(layer_name, targets)
119 + or _find_first_match(module.__class__.__name__, targets, True)
120 + or _match_fused_layer(layer_name, targets, fused_mapping))
121
122 if matched_target is None:
123 raise ValueError(
@@ -205,11 +169,19 @@ def _is_equal_or_regex_match(value: str,
169 return False
170
171
208 -def _match_fused_layer(layer_name: str,
209 - target_layers: Iterable[str]) -> Optional[str]:
172 +def _match_fused_layer(
173 + layer_name: str, target_layers: Iterable[str],
174 + fused_mapping: Mapping[str, List[str]]) -> Optional[str]:
175 """
176 Match a fused layer name to its corresponding individual layer in
212 - target_layers.
177 + target_layers. Returns first value in fused_mapping which matches targets
178 +
179 + Implements an "all" matching strategy where a fused layer matches iff
180 + "all" of its components match
181 +
182 + :param layer_name: layer name
183 + :param target_layers: list of targets to match the layer against
184 + :param fused_mapping: map from fused layer names to its components
185
186 Examples:
187 layer_name = "model.layers.0.self_attn.qkv_proj"
@@ -217,27 +189,25 @@ def _match_fused_layer(layer_name: str,
189 "model.layers.0.self_attn.k_proj",
190 "model.layers.0.self_attn.v_proj"]
191 """
220 - # Split into parent path and layer type
221 - # e.g., "model.layers.0.self_attn" and "qkv_proj"
222 - parent_path = ".".join(layer_name.split(".")[:-1])
223 - layer_type = layer_name.split(".")[-1]
224 -
225 - if layer_type not in FUSED_LAYER_NAME_MAPPING:
192 + # find layer_name in mapping
193 + fused = next((key for key in fused_mapping if layer_name.endswith(key)),
194 + None)
195 + if fused is None:
196 return None
197
228 - possible_layer_types = FUSED_LAYER_NAME_MAPPING[layer_type]
229 -
230 - # Look for a target layer that:
231 - # 1. Has the same parent path
232 - # 2. Ends with one of the possible individual layer types
233 - for target in target_layers:
234 - is_same_parent = parent_path in target
235 - is_matching_type = any(type_suffix in target
236 - for type_suffix in possible_layer_types)
237 -
238 - if is_same_parent and is_matching_type and all(
239 - (f"{parent_path}.{type_suffix}" in target_layers)
240 - for type_suffix in possible_layer_types):
241 - return target
198 + # expand path of unfused components
199 + unfused_paths = [
200 + layer_name.replace(fused, unfused) for unfused in fused_mapping[fused]
201 + ]
202
243 - return None
203 + # for each unfused component, find a match in targets
204 + unfused_matches: List[Optional[str]] = []
205 + for unfused in unfused_paths:
206 + for target in target_layers:
207 + if _is_equal_or_regex_match(unfused, target):
208 + unfused_matches.append(target)
209 + break
210 + else:
211 + unfused_matches.append(None)
212 +
213 + return unfused_matches[0] if all(unfused_matches) else None
vllm/model_executor/layers/quantization/quark/quark.py
+5 -5
@@ -18,8 +18,6 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
18 QuarkScheme, QuarkW8A8Fp8, QuarkW8A8Int8)
19 from vllm.model_executor.layers.quantization.quark.utils import (
20 deep_compare, should_ignore_layer)
21 -from vllm.model_executor.layers.quantization.utils.quant_utils import (
22 - FUSED_LAYER_NAME_MAPPING)
21 from vllm.platforms import current_platform
22
23 __all__ = ["QuarkLinearMethod"]
@@ -58,7 +56,9 @@ class QuarkConfig(QuantizationConfig):
56
57 # Check if the layer is skipped for quantization.
58 exclude_layers = cast(List[str], self.quant_config.get("exclude"))
61 - if should_ignore_layer(prefix, ignore=exclude_layers):
59 + if should_ignore_layer(prefix,
60 + ignore=exclude_layers,
61 + fused_mapping=self.packed_modules_mapping):
62 return UnquantizedLinearMethod()
63 if isinstance(layer, LinearBase):
64 scheme = self.get_scheme(layer=layer, layer_name=prefix)
@@ -201,8 +201,8 @@ class QuarkConfig(QuantizationConfig):
201 module: torch.nn.Module) -> Dict[str, Any]:
202
203 proj_name = layer_name.split(".")[-1]
204 - if proj_name in FUSED_LAYER_NAME_MAPPING:
205 - shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
204 + if proj_name in self.packed_modules_mapping:
205 + shard_proj_names = self.packed_modules_mapping[proj_name]
206
207 # Convert fused_name --> [shard_names]
208 shard_names = [
vllm/model_executor/layers/quantization/quark/utils.py
+9 -8
@@ -1,10 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import re
4 -from typing import Any, Iterable, Optional
5 -
6 -from vllm.model_executor.layers.quantization.utils.quant_utils import (
7 - FUSED_LAYER_NAME_MAPPING)
4 +from types import MappingProxyType
5 +from typing import Any, Iterable, List, Mapping, Optional
6
7
8 def deep_compare(dict1: Any, dict2: Any) -> bool:
@@ -20,8 +18,11 @@ def deep_compare(dict1: Any, dict2: Any) -> bool:
18 return dict1 == dict2
19
20
23 -def should_ignore_layer(layer_name: Optional[str],
24 - ignore: Iterable[str]) -> bool:
21 +def should_ignore_layer(
22 + layer_name: Optional[str],
23 + ignore: Iterable[str],
24 + fused_mapping: Mapping[str, List[str]] = MappingProxyType({})
25 +) -> bool:
26 if layer_name is None:
27 return False
28
@@ -33,8 +34,8 @@ def should_ignore_layer(layer_name: Optional[str],
34 # in the safetensors checkpoint. So, we convert the name
35 # from the fused version to unfused + check to make sure that
36 # each shard of the fused layer has the same scheme.
36 - if proj_name in FUSED_LAYER_NAME_MAPPING:
37 - shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
37 + if proj_name in fused_mapping:
38 + shard_proj_names = fused_mapping[proj_name]
39
40 # Convert fused_name --> [shard_names]
41 shard_names = [
vllm/model_executor/layers/quantization/utils/quant_utils.py
+14 -12
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 """This file is used for /tests and /benchmarks"""
3 -from typing import List, Optional, Tuple
3 +from types import MappingProxyType
4 +from typing import List, Mapping, Optional, Tuple
5
6 import numpy
7 import torch
@@ -12,14 +13,6 @@ from vllm.scalar_type import ScalarType, scalar_types
13 SUPPORTED_GPTQ_QUANT_TYPES = [scalar_types.uint4b8, scalar_types.uint8b128]
14 SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
15
15 -# Note: this is a hack. We should update each model to register the
16 -# stacked params and get it from there instead in a future PR.
17 -# fused_name: List[shard_name]
18 -FUSED_LAYER_NAME_MAPPING = {
19 - "qkv_proj": ["q_proj", "k_proj", "v_proj"],
20 - "gate_up_proj": ["gate_proj", "up_proj"]
21 -}
22 -
16
17 # Normalize the group_shape to the full extent for any dims that are -1
18 def _normalize_quant_group_shape(x: torch.Tensor, group_shape: Tuple[int,
@@ -178,14 +171,23 @@ def unpack_quantized_values_into_int32(w_q: torch.Tensor,
171 return res.permute(inv_perm)
172
173
181 -def is_layer_skipped(prefix: str, ignored_layers: List[str]) -> bool:
174 +def is_layer_skipped(
175 + prefix: str,
176 + ignored_layers: List[str],
177 + fused_mapping: Mapping[str, List[str]] = MappingProxyType({})
178 +) -> bool:
179 # prefix: model.layers.0.self_attn.q_proj
180 # proj_name: q_proj
181 proj_name = prefix.split(".")[-1]
185 - if proj_name in FUSED_LAYER_NAME_MAPPING:
182 +
183 + # Fused layers like gate_up_proj or qkv_proj will not be fused
184 + # in the safetensors checkpoint. So, we convert the name
185 + # from the fused version to unfused + check to make sure that
186 + # each shard of the fused layer has the same scheme.
187 + if proj_name in fused_mapping:
188 shard_prefixes = [
189 prefix.replace(proj_name, shard_proj_name)
188 - for shard_proj_name in FUSED_LAYER_NAME_MAPPING[proj_name]
190 + for shard_proj_name in fused_mapping[proj_name]
191 ]
192
193 is_skipped = None
vllm/model_executor/model_loader/loader.py
+4
@@ -43,6 +43,7 @@ from vllm.model_executor.model_loader.tensorizer import (
43 TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
44 serialize_vllm_model, tensorizer_weights_iterator)
45 from vllm.model_executor.model_loader.utils import (ParamMapping,
46 + configure_quant_config,
47 get_model_architecture,
48 set_default_torch_dtype)
49 from vllm.model_executor.model_loader.weight_utils import (
@@ -113,6 +114,9 @@ def _initialize_model(
114 model_config = vllm_config.model_config
115 model_class, _ = get_model_architecture(model_config)
116
117 + if vllm_config.quant_config is not None:
118 + configure_quant_config(vllm_config.quant_config, model_class)
119 +
120 signatures = inspect.signature(model_class.__init__)
121 all_params = [param.name for param in signatures.parameters.values()]
122 if "vllm_config" in all_params and "prefix" in all_params:
vllm/model_executor/model_loader/utils.py
+22
@@ -11,6 +11,8 @@ from transformers.dynamic_module_utils import get_class_from_dynamic_module
11
12 from vllm.config import ModelConfig, ModelImpl
13 from vllm.logger import init_logger
14 +from vllm.model_executor.layers.quantization.base_config import (
15 + QuantizationConfig)
16 from vllm.model_executor.models import ModelRegistry
17 from vllm.model_executor.models.adapters import (as_classification_model,
18 as_embedding_model,
@@ -138,3 +140,23 @@ class ParamMapping:
140 if module_name.endswith(key):
141 return key, value
142 return None
143 +
144 +
145 +def configure_quant_config(quant_config: QuantizationConfig,
146 + model_class: Type[nn.Module]):
147 + """
148 + Pass packed_modules_mapping by reference to quant_config so that
149 + quant_config can properly match fused modules
150 +
151 + Note that model attributes are passed by reference to quant_config,
152 + enabling them to be updated by model_class.__new__ (ex. chatglm, qwen)
153 + """
154 + packed_mapping = getattr(model_class, "packed_modules_mapping", None)
155 + if packed_mapping is not None:
156 + # pass packed_modules_mapping by reference to quant_config
157 + quant_config.packed_modules_mapping = packed_mapping
158 + else:
159 + logger.warning(
160 + "The model class %s has not defined `packed_modules_mapping`, "
161 + "this may lead to incorrect mapping of quantized or ignored "
162 + "modules", model_class.__name__)
vllm/model_executor/models/chatglm.py
+21 -5
@@ -265,12 +265,14 @@ class GLMAttention(nn.Module):
265 self.total_num_kv_heads,
266 bias=config.add_bias_linear or config.add_qkv_bias,
267 quant_config=quant_config,
268 + prefix=f"{prefix}.query_key_value",
269 )
270 self.dense = RowParallelLinear(
271 self.total_num_heads * self.head_dim,
272 config.hidden_size,
273 bias=config.add_bias_linear,
274 quant_config=quant_config,
275 + prefix=f"{prefix}.dense",
276 )
277
278 # https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
@@ -327,6 +329,7 @@ class GLMMLP(nn.Module):
329 self,
330 config: ChatGLMConfig,
331 quant_config: Optional[QuantizationConfig] = None,
332 + prefix: str = "",
333 ):
334 super().__init__()
335
@@ -338,6 +341,7 @@ class GLMMLP(nn.Module):
341 [config.ffn_hidden_size] * 2,
342 bias=config.add_bias_linear,
343 quant_config=quant_config,
344 + prefix=f"{prefix}.dense_h_to_4h",
345 )
346
347 self.activation_func = SiluAndMul()
@@ -348,6 +352,7 @@ class GLMMLP(nn.Module):
352 config.hidden_size,
353 bias=config.add_bias_linear,
354 quant_config=quant_config,
355 + prefix=f"{prefix}.dense_4h_to_h",
356 )
357
358 def forward(self, hidden_states):
@@ -396,7 +401,7 @@ class GLMBlock(nn.Module):
401 config.hidden_size, eps=config.layernorm_epsilon)
402
403 # MLP
399 - self.mlp = GLMMLP(config, quant_config)
404 + self.mlp = GLMMLP(config, quant_config, prefix=f"{prefix}.mlp")
405
406 def forward(
407 self,
@@ -507,7 +512,8 @@ class ChatGLMModel(nn.Module):
512
513 self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
514 config.hidden_size,
510 - quant_config=quant_config)
515 + quant_config=quant_config,
516 + prefix=f"{prefix}.embedding")
517
518 self.num_layers = config.num_layers
519 self.multi_query_group_num = config.multi_query_group_num
@@ -766,6 +772,7 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
772 SupportsMultiModal):
773 # Ensure that the LoRA support check passes when the class is not
774 # initialized, but set all these attributes to empty.
775 + # These will be updated when an instance class is selected
776 packed_modules_mapping = {}
777 supported_lora_modules = []
778 embedding_modules = {}
@@ -777,9 +784,18 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
784 prefix: str = "",
785 ) -> None:
786 config = vllm_config.model_config.hf_config
787 +
788 # Initialize VL
781 - if hasattr(config, "vision_config"):
782 - return ChatGLMV(vllm_config=vllm_config, prefix=prefix)
789 + if hasattr(config, "vision_config"): # noqa: SIM108
790 + instance_cls = ChatGLMV
791 # Initialize LLM
792 else:
785 - return ChatGLM(vllm_config=vllm_config, prefix=prefix)
\ No newline at end of file
793 + instance_cls = ChatGLM
794 +
795 + # quant_config references base class members,
796 + # so update values before init is called
797 + cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
798 + cls.supported_lora_modules += instance_cls.supported_lora_modules
799 + cls.embedding_modules.update(instance_cls.embedding_modules)
800 + cls.embedding_padding_modules += instance_cls.embedding_padding_modules
801 + return instance_cls(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/glm4_vision_encoder.py
+22 -9
@@ -74,11 +74,13 @@ class Attention(nn.Module):
74 self.head_dim,
75 config.num_heads,
76 quant_config=quant_config,
77 + prefix=f"{prefix}.query_key_value",
78 )
79 self.dense = RowParallelLinear(
80 config.hidden_size,
81 config.hidden_size,
82 quant_config=quant_config,
83 + prefix=f"{prefix}.dense",
84 )
85
86 self.attn = MultiHeadAttention(self.num_heads_per_rank, self.head_dim,
@@ -101,6 +103,7 @@ class MLP(nn.Module):
103 self,
104 config,
105 quant_config: Optional[QuantizationConfig] = None,
106 + prefix: str = '',
107 ):
108 super().__init__()
109 self.config = config
@@ -109,11 +112,13 @@ class MLP(nn.Module):
112 config.hidden_size,
113 config.intermediate_size,
114 quant_config=quant_config,
115 + prefix=f"{prefix}.fc1",
116 )
117 self.fc2 = RowParallelLinear(
118 config.intermediate_size,
119 config.hidden_size,
120 quant_config=quant_config,
121 + prefix=f"{prefix}.fc2",
122 )
123
124 def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -137,7 +142,9 @@ class TransformerLayer(nn.Module):
142 self.attention = Attention(config,
143 quant_config=quant_config,
144 prefix=f"{prefix}.attention")
140 - self.mlp = MLP(config, quant_config=quant_config)
145 + self.mlp = MLP(config,
146 + quant_config=quant_config,
147 + prefix=f"{prefix}.mlp")
148 self.post_attention_layernorm = LayerNorm(config.hidden_size,
149 eps=config.layer_norm_eps)
150
@@ -164,7 +171,7 @@ class Transformer(nn.Module):
171 self.layers = nn.ModuleList([
172 TransformerLayer(config,
173 quant_config=quant_config,
167 - prefix=f"{prefix}.layer.{layer_idx}")
174 + prefix=f"{prefix}.layers.{layer_idx}")
175 for layer_idx in range(config.num_hidden_layers)
176 ])
177
@@ -181,6 +188,7 @@ class GLU(nn.Module):
188 config,
189 in_features,
190 quant_config: Optional[QuantizationConfig] = None,
191 + prefix: str = '',
192 ):
193 """
194 The original implementation is the same as:
@@ -222,7 +230,8 @@ class GLU(nn.Module):
230 self.linear_proj = ReplicatedLinear(in_features,
231 config.hidden_size,
232 bias=False,
225 - quant_config=quant_config)
233 + quant_config=quant_config,
234 + prefix=f"{prefix}.linear_proj")
235 self.norm1 = nn.LayerNorm(config.hidden_size)
236 self.act1 = nn.GELU()
237 self.act2 = SiluAndMul()
@@ -230,12 +239,15 @@ class GLU(nn.Module):
239 self.merged_proj = MergedColumnParallelLinear(
240 config.hidden_size, [config.ffn_hidden_size] * 2,
241 bias=False,
233 - quant_config=quant_config)
242 + quant_config=quant_config,
243 + prefix=f"{prefix}.merged_proj")
244
235 - self.dense_4h_to_h = RowParallelLinear(config.ffn_hidden_size,
236 - config.hidden_size,
237 - bias=False,
238 - quant_config=quant_config)
245 + self.dense_4h_to_h = RowParallelLinear(
246 + config.ffn_hidden_size,
247 + config.hidden_size,
248 + bias=False,
249 + quant_config=quant_config,
250 + prefix=f"{prefix}.dense_4h_to_h")
251
252 def forward(self, x):
253 x, _ = self.linear_proj(x)
@@ -262,7 +274,8 @@ class EVA2CLIPModel(nn.Module):
274 prefix=f"{prefix}.transformer")
275 self.linear_proj = GLU(config,
276 in_features=config.hidden_size,
265 - quant_config=quant_config)
277 + quant_config=quant_config,
278 + prefix=f"{prefix}.linear_proj")
279 self.conv = nn.Conv2d(in_channels=vision_config.hidden_size,
280 out_channels=config.hidden_size,
281 kernel_size=2,
vllm/model_executor/models/minicpmv.py
+11 -3
@@ -1473,6 +1473,7 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
1473 """
1474 # Ensure that the LoRA support check passes when the class is not
1475 # initialized, but set all these attributes to empty.
1476 + # These will be updated when an instance class is selected
1477 packed_modules_mapping = {}
1478 supported_lora_modules = []
1479 embedding_modules = {}
@@ -1489,8 +1490,15 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
1490 version = str(config.version).split(".")
1491 version = tuple([int(x) for x in version])
1492 # Dispatch class based on version
1492 - instance_class = _SUPPORT_VERSION.get(version)
1493 - if instance_class is None:
1493 + instance_cls = _SUPPORT_VERSION.get(version)
1494 + if instance_cls is None:
1495 raise ValueError(
1496 "Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
1496 - return instance_class(vllm_config=vllm_config, prefix=prefix)
1497 +
1498 + # quant_config references base class members,
1499 + # so update values before init is called
1500 + cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
1501 + cls.supported_lora_modules += instance_cls.supported_lora_modules
1502 + cls.embedding_modules.update(instance_cls.embedding_modules)
1503 + cls.embedding_padding_modules += instance_cls.embedding_padding_modules
1504 + return instance_cls(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/qwen.py
+13 -3
@@ -1135,6 +1135,7 @@ class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1135 """
1136 # Ensure that the LoRA support check passes when the class is not
1137 # initialized, but set all these attributes to empty.
1138 + # These will be updated when an instance class is selected
1139 packed_modules_mapping = {}
1140 supported_lora_modules = []
1141 embedding_modules = {}
@@ -1146,9 +1147,18 @@ class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1147 prefix: str = "",
1148 ) -> QWenBaseModel:
1149 config = vllm_config.model_config.hf_config
1150 +
1151 # Initialize VL
1150 - if hasattr(config, "visual"):
1151 - return QWenVL(vllm_config=vllm_config, prefix=prefix)
1152 + if hasattr(config, "visual"): # noqa: SIM108
1153 + instance_cls = QWenVL
1154 # Initialize LLM
1155 else:
1154 - return QWenLLM(vllm_config=vllm_config, prefix=prefix)
1156 + instance_cls = QWenLLM
1157 +
1158 + # quant_config references base class members,
1159 + # so update values before init is called
1160 + cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
1161 + cls.supported_lora_modules += instance_cls.supported_lora_modules
1162 + cls.embedding_modules.update(instance_cls.embedding_modules)
1163 + cls.embedding_padding_modules += instance_cls.embedding_padding_modules
1164 + return instance_cls(vllm_config=vllm_config, prefix=prefix)