A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Quantization] Add field to skip unquantized modules for GPTQ config (#25455)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Sep 26, 2025 at 23:47 UTC
d4d9899860e75b2ef4aa6dda205c23e641876d24
16 files changed
+219
-153
vllm/config/__init__.py
+1
@@ -270,6 +270,7 @@ class VllmConfig:
270
f"{model_config.dtype} is not supported for quantization "
271
f"method {model_config.quantization}. Supported dtypes: "
272
f"{supported_dtypes}")
273
+ quant_config.maybe_update_config(model_config.model)
274
return quant_config
275
return None
276
vllm/model_executor/layers/quantization/base_config.py
+6
@@ -162,3 +162,9 @@ class QuantizationConfig(ABC):
162
"""
163
# TODO (@kylesayrs): add implementations for all subclasses
164
pass
165
+
166
+ def maybe_update_config(self, model_name: str): # noqa: B027
167
+ """
168
+ Interface to update values after config initialization.
169
+ """
170
+ pass
vllm/model_executor/layers/quantization/gptq.py
+46
-6
@@ -7,6 +7,7 @@ from fractions import Fraction
7
from typing import Any, Optional, Union
8
9
import torch
10
+from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
11
from torch.nn.parameter import Parameter
12
13
from vllm import _custom_ops as ops
@@ -22,6 +23,8 @@ from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
23
PackedColumnParameter,
24
PackedvLLMParameter,
25
RowvLLMParameter)
26
+from vllm.transformers_utils.config import get_safetensors_params_metadata
27
+from vllm.utils import is_list_of
28
29
30
class GPTQConfig(QuantizationConfig):
@@ -38,6 +41,7 @@ class GPTQConfig(QuantizationConfig):
41
lm_head_quantized: bool,
42
dynamic: dict[str, dict[str, Union[int, bool]]],
43
autoround_version: str = "",
44
+ modules_in_block_to_quantize: Optional[list[str]] = None,
45
) -> None:
46
# GPTQModel use `dynamic` config property to allow per module
47
# quantization config so each module can be individually optimized.
@@ -75,15 +79,20 @@ class GPTQConfig(QuantizationConfig):
79
"Currently, only 2/3/4/8-bit weight quantization is "
80
f"supported for GPTQ, but got {self.weight_bits} bits.")
81
82
+ self.modules_in_block_to_quantize = modules_in_block_to_quantize or []
83
+
84
# used to identify GPTQ model quantized by autoround
85
self.autoround_version = autoround_version
86
87
def __repr__(self) -> str:
82
- return (f"GPTQConfig(weight_bits={self.weight_bits}, "
83
- f"group_size={self.group_size}, "
84
- f"desc_act={self.desc_act}), "
85
- f"lm_head_quantized={self.lm_head_quantized}), "
86
- f"dynamic={self.dynamic}")
88
+ return (
89
+ f"GPTQConfig(weight_bits={self.weight_bits}, "
90
+ f"group_size={self.group_size}, "
91
+ f"desc_act={self.desc_act}), "
92
+ f"lm_head_quantized={self.lm_head_quantized}, "
93
+ f"dynamic={self.dynamic}, "
94
+ f"modules_in_block_to_quantize={self.modules_in_block_to_quantize})"
95
+ )
96
97
@classmethod
98
def get_name(cls) -> QuantizationMethods:
@@ -114,8 +123,10 @@ class GPTQConfig(QuantizationConfig):
123
default=False)
124
autoround_version = cls.get_from_keys_or(config, ["autoround_version"],
125
default="")
126
+ modules_in_block_to_quantize = cls.get_from_keys_or(
127
+ config, ["modules_in_block_to_quantize"], default=None)
128
return cls(weight_bits, group_size, desc_act, lm_head_quantized,
118
- dynamic, autoround_version)
129
+ dynamic, autoround_version, modules_in_block_to_quantize)
130
131
def get_quant_method(
132
self, layer: torch.nn.Module, prefix: str
@@ -136,6 +147,35 @@ class GPTQConfig(QuantizationConfig):
147
148
return get_linear_quant_method(self, layer, prefix, GPTQLinearMethod)
149
150
+ def apply_vllm_mapper(self, hf_to_vllm_mapper):
151
+ if self.modules_in_block_to_quantize is not None:
152
+ self.modules_in_block_to_quantize = hf_to_vllm_mapper.apply_list(
153
+ self.modules_in_block_to_quantize)
154
+
155
+ def maybe_update_config(self,
156
+ model_name: str,
157
+ revision: Optional[str] = None):
158
+ if self.modules_in_block_to_quantize:
159
+ if is_list_of(self.modules_in_block_to_quantize, list):
160
+ # original modules_in_block_to_quantize: list[list[str]]
161
+ # flatten original modules_in_block_to_quantize
162
+ self.modules_in_block_to_quantize = [
163
+ item for sublist in self.modules_in_block_to_quantize
164
+ for item in sublist
165
+ ]
166
+ return
167
+
168
+ unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]
169
+ metadata = get_safetensors_params_metadata(model_name,
170
+ revision=revision)
171
+ quant_layers: set[str] = {
172
+ param_name.rsplit(".", 1)[0]
173
+ for param_name, info in metadata.items()
174
+ if (dtype := info.get('dtype', None))
175
+ and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes
176
+ }
177
+ self.modules_in_block_to_quantize = list(quant_layers)
178
+
179
180
class ExllamaState(Enum):
181
vllm/model_executor/layers/quantization/gptq_marlin.py
+55
-10
@@ -5,6 +5,7 @@ from copy import deepcopy
5
from typing import Any, Callable, Optional, Union
6
7
import torch
8
+from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
9
10
import vllm.model_executor.layers.fused_moe # noqa
11
from vllm import _custom_ops as ops
@@ -35,6 +36,8 @@ from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
36
RowvLLMParameter)
37
from vllm.platforms import current_platform
38
from vllm.scalar_type import scalar_types
39
+from vllm.transformers_utils.config import get_safetensors_params_metadata
40
+from vllm.utils import is_list_of
41
42
logger = init_logger(__name__)
43
@@ -71,10 +74,16 @@ class GPTQMarlinConfig(QuantizationConfig):
74
(8, True): scalar_types.uint8b128,
75
}
76
74
- def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
75
- is_sym: bool, lm_head_quantized: bool,
76
- dynamic: dict[str, dict[str, Union[int, bool]]],
77
- full_config: dict[str, Any]) -> None:
77
+ def __init__(
78
+ self,
79
+ weight_bits: int,
80
+ group_size: int,
81
+ desc_act: bool,
82
+ is_sym: bool,
83
+ lm_head_quantized: bool,
84
+ dynamic: dict[str, dict[str, Union[int, bool]]],
85
+ full_config: dict[str, Any],
86
+ modules_in_block_to_quantize: Optional[list[str]] = None) -> None:
87
super().__init__()
88
if desc_act and group_size == -1:
89
# In this case, act_order == True is the same as act_order == False
@@ -121,15 +130,19 @@ class GPTQMarlinConfig(QuantizationConfig):
130
131
self.quant_type = self.TYPE_MAP[(weight_bits, is_sym)]
132
133
+ self.modules_in_block_to_quantize = modules_in_block_to_quantize or []
134
# used to identify GPTQ model quantized by autoround
135
self.autoround_version = full_config.get("autoround_version", "")
136
137
def __repr__(self) -> str:
128
- return (f"GPTQMarlinConfig(quant_type={self.quant_type}, "
129
- f"group_size={self.group_size}, "
130
- f"desc_act={self.desc_act}, "
131
- f"lm_head_quantized={self.lm_head_quantized}), "
132
- f"dynamic={self.dynamic}")
138
+ return (
139
+ f"GPTQMarlinConfig(quant_type={self.quant_type}, "
140
+ f"group_size={self.group_size}, "
141
+ f"desc_act={self.desc_act}, "
142
+ f"lm_head_quantized={self.lm_head_quantized}, "
143
+ f"dynamic={self.dynamic}, "
144
+ f"modules_in_block_to_quantize={self.modules_in_block_to_quantize})"
145
+ )
146
147
@classmethod
148
def get_name(cls) -> QuantizationMethods:
@@ -158,8 +171,11 @@ class GPTQMarlinConfig(QuantizationConfig):
171
is_sym = cls.get_from_keys(config, ["sym"])
172
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
173
default=False)
174
+ modules_in_block_to_quantize = cls.get_from_keys_or(
175
+ config, ["modules_in_block_to_quantize"], default=None)
176
return cls(weight_bits, group_size, desc_act, is_sym,
162
- lm_head_quantized, dynamic, config)
177
+ lm_head_quantized, dynamic, config,
178
+ modules_in_block_to_quantize)
179
180
@classmethod
181
def override_quantization_method(
@@ -223,6 +239,35 @@ class GPTQMarlinConfig(QuantizationConfig):
239
return check_marlin_supported(quant_type=cls.TYPE_MAP[(num_bits, sym)],
240
group_size=group_size)
241
242
+ def apply_vllm_mapper(self, hf_to_vllm_mapper):
243
+ if self.modules_in_block_to_quantize is not None:
244
+ self.modules_in_block_to_quantize = hf_to_vllm_mapper.apply_list(
245
+ self.modules_in_block_to_quantize)
246
+
247
+ def maybe_update_config(self,
248
+ model_name: str,
249
+ revision: Optional[str] = None):
250
+ if self.modules_in_block_to_quantize:
251
+ if is_list_of(self.modules_in_block_to_quantize, list):
252
+ # original modules_in_block_to_quantize: list[list[str]]
253
+ # flatten original modules_in_block_to_quantize
254
+ self.modules_in_block_to_quantize = [
255
+ item for sublist in self.modules_in_block_to_quantize
256
+ for item in sublist
257
+ ]
258
+ return
259
+
260
+ unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]
261
+ metadata = get_safetensors_params_metadata(model_name,
262
+ revision=revision)
263
+ quant_layers: set[str] = {
264
+ param_name.rsplit(".", 1)[0]
265
+ for param_name, info in metadata.items()
266
+ if (dtype := info.get('dtype', None))
267
+ and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes
268
+ }
269
+ self.modules_in_block_to_quantize = list(quant_layers)
270
+
271
272
class GPTQMarlinLinearMethod(LinearMethodBase):
273
"""Linear method for GPTQ Marlin.
vllm/model_executor/layers/quantization/utils/gptq_utils.py
+51
-1
@@ -1,7 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from collections.abc import Mapping
4
from copy import deepcopy
5
from fractions import Fraction
6
+from types import MappingProxyType
7
from typing import Optional, Union
8
9
import regex as re
@@ -70,6 +72,49 @@ def get_dynamic_override(
72
return default_value
73
74
75
+def is_layer_gptq_quantized(
76
+ prefix: str,
77
+ quantized_layers: list[str],
78
+ fused_mapping: Mapping[str, list[str]] = MappingProxyType({})
79
+) -> bool:
80
+ # prefix: model.layers.0.self_attn.q_proj
81
+ # proj_name: q_proj
82
+
83
+ # GPTQ's `modules_in_block_to_quantize`:
84
+ # Substr: ["self_attn.k_proj", "self_attn.v_proj", "self_attn.q_proj"]
85
+ # Full prefix ["model.layers.0.self_attn.q_proj"]
86
+
87
+ proj_name = prefix.split(".")[-1]
88
+
89
+ # Fused layers like gate_up_proj or qkv_proj will not be fused
90
+ # in the safetensors checkpoint. So, we convert the name
91
+ # from the fused version to unfused + check to make sure that
92
+ # each shard of the fused layer has the same scheme.
93
+ if proj_name in fused_mapping:
94
+ shard_prefixes = [
95
+ prefix.replace(proj_name, shard_proj_name)
96
+ for shard_proj_name in fused_mapping[proj_name]
97
+ ]
98
+
99
+ is_quantized = None
100
+ for shard_prefix in shard_prefixes:
101
+ is_shard_quantized = any(layer in shard_prefix
102
+ for layer in quantized_layers)
103
+
104
+ if is_quantized is None:
105
+ is_quantized = is_shard_quantized
106
+ elif is_shard_quantized != is_quantized:
107
+ raise ValueError(
108
+ f"Detected some but not all shards of {prefix} "
109
+ "are quantized. All shards of fused layers "
110
+ "to have the same precision.")
111
+ else:
112
+ is_quantized = any(layer in prefix for layer in quantized_layers)
113
+
114
+ assert is_quantized is not None
115
+ return is_quantized
116
+
117
+
118
def get_linear_quant_method(
119
config: QuantizationConfig,
120
layer: torch.nn.Module,
@@ -80,10 +125,15 @@ def get_linear_quant_method(
125
parallel_lm_head_quantized = isinstance(
126
layer, ParallelLMHead) and cloned_config.lm_head_quantized
127
if isinstance(layer, LinearBase) or parallel_lm_head_quantized:
128
+ is_layer_quantized = is_layer_gptq_quantized(
129
+ prefix=prefix,
130
+ quantized_layers=cloned_config.modules_in_block_to_quantize,
131
+ fused_mapping=cloned_config.packed_modules_mapping)
132
# False = skip module, None = no override, else = Positive match
133
if get_dynamic_override( # noqa: E712
134
cloned_config, # noqa: E712
86
- layer_name=prefix) == False: # noqa: E712
135
+ layer_name=prefix) == False or (
136
+ not is_layer_quantized): # noqa: E712
137
if parallel_lm_head_quantized:
138
return UnquantizedEmbeddingMethod()
139
return UnquantizedLinearMethod()
vllm/model_executor/models/keye.py
+2
-10
@@ -25,9 +25,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
25
QKVParallelLinear,
26
RowParallelLinear)
27
from vllm.model_executor.layers.quantization import QuantizationConfig
28
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
29
-from vllm.model_executor.layers.quantization.gptq_marlin import (
30
- GPTQMarlinConfig)
28
from vllm.model_executor.model_loader.weight_utils import (
29
default_weight_loader, maybe_remap_kv_scale_name)
30
from vllm.model_executor.models.module_mapping import MultiModelKeys
@@ -1281,11 +1278,6 @@ class BaseKeyeModule(nn.Module):
1278
1279
raise ValueError("Only image or video modality is supported")
1280
1284
- def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
1285
- if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
1286
- return None
1287
- return quant_config
1288
-
1281
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1282
super().__init__()
1283
config: PretrainedConfig = vllm_config.model_config.hf_config
@@ -1297,14 +1289,14 @@ class BaseKeyeModule(nn.Module):
1289
1290
self.visual = KeyeSiglipVisionModel(
1291
config.vision_config,
1300
- quant_config=self._maybe_ignore_quant_config(quant_config),
1292
+ quant_config=quant_config,
1293
prefix=maybe_prefix(prefix, "visual"),
1294
)
1295
1296
self.mlp_AR = self._build_projector(
1297
config,
1298
config.vision_config,
1307
- quant_config=self._maybe_ignore_quant_config(quant_config),
1299
+ quant_config=quant_config,
1300
prefix=maybe_prefix(prefix, "mlp_AR"),
1301
)
1302
vllm/model_executor/models/minicpmo.py
+1
-35
@@ -28,7 +28,7 @@ from typing import Annotated, Any, Callable, Literal, Optional, Union
28
29
import torch
30
from torch import nn
31
-from transformers import BatchFeature, PretrainedConfig
31
+from transformers import BatchFeature
32
from transformers.modeling_outputs import BaseModelOutputWithPast
33
from transformers.models.whisper.modeling_whisper import (ACT2FN,
34
WhisperAttention,
@@ -36,10 +36,6 @@ from transformers.models.whisper.modeling_whisper import (ACT2FN,
36
WhisperEncoder)
37
38
from vllm.config import VllmConfig
39
-from vllm.model_executor.layers.quantization import QuantizationConfig
40
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
41
-from vllm.model_executor.layers.quantization.gptq_marlin import (
42
- GPTQMarlinConfig)
39
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
40
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
41
NestedTensors)
@@ -548,36 +544,6 @@ class MiniCPMO(MiniCPMV2_6):
544
545
self.audio_token_id = None
546
551
- def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
552
- # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
553
- # seems to avoid vision encoder sections for some models.
554
- # See: https://huggingface.co/openbmb/MiniCPM-o-2_6-int4
555
- if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
556
- return None
557
- return quant_config
558
-
559
- def init_vision_module(
560
- self,
561
- config: PretrainedConfig,
562
- quant_config: Optional[QuantizationConfig] = None,
563
- prefix: str = "",
564
- ) -> nn.Module:
565
- # MiniCPMO GPTQ model leave vpm unquantized.
566
- quant_config = self._maybe_ignore_quant_config(quant_config)
567
- return super().init_vision_module(config, quant_config, prefix)
568
-
569
- def init_resampler(
570
- self,
571
- embed_dim: int,
572
- vision_dim: int,
573
- quant_config: Optional[QuantizationConfig] = None,
574
- prefix: str = "",
575
- ) -> nn.Module:
576
- # MiniCPMO GPTQ model leave resampler unquantized.
577
- quant_config = self._maybe_ignore_quant_config(quant_config)
578
- return super().init_resampler(embed_dim, vision_dim, quant_config,
579
- prefix)
580
-
547
def init_audio_module(self, *, vllm_config: VllmConfig, prefix: str = ""):
548
# Do not use parameters temporarily
549
audio_config = self.config.audio_config
vllm/model_executor/models/ovis.py
+1
-12
@@ -31,9 +31,6 @@ from vllm.config import VllmConfig
31
from vllm.model_executor.layers.linear import ReplicatedLinear
32
from vllm.model_executor.layers.quantization.base_config import (
33
QuantizationConfig)
34
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
35
-from vllm.model_executor.layers.quantization.gptq_marlin import (
36
- GPTQMarlinConfig)
34
from vllm.model_executor.models.aimv2 import AIMv2Model
35
from vllm.model_executor.models.siglip import SiglipVisionModel
36
from vllm.model_executor.models.utils import (AutoWeightsLoader, flatten_bn,
@@ -416,7 +413,7 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP):
413
414
self.visual_tokenizer = VisualTokenizer(
415
config=config.visual_tokenizer_config,
419
- quant_config=self._maybe_ignore_quant_config(quant_config),
416
+ quant_config=quant_config,
417
prefix=f"{prefix}.visual_tokenizer",
418
)
419
@@ -430,14 +427,6 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP):
427
self.make_empty_intermediate_tensors = (
428
self.get_language_model().make_empty_intermediate_tensors)
429
433
- def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
434
- # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
435
- # seems to avoid vision encoder sections for some models.
436
- # See: https://huggingface.co/AIDC-AI/Ovis2-2B-GPTQ-Int4
437
- if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
438
- return None
439
- return quant_config
440
-
430
def _parse_and_validate_image_input(
431
self, **kwargs: object) -> Optional[OvisImagePatchInputs]:
432
pixel_values = kwargs.pop("pixel_values", None)
vllm/model_executor/models/qwen2_5_vl.py
+1
-12
@@ -52,9 +52,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
52
RowParallelLinear)
53
# yapf: enable
54
from vllm.model_executor.layers.quantization import QuantizationConfig
55
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
56
-from vllm.model_executor.layers.quantization.gptq_marlin import (
57
- GPTQMarlinConfig)
55
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56
from vllm.model_executor.models.module_mapping import MultiModelKeys
57
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -1015,8 +1012,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1012
self.visual = Qwen2_5_VisionTransformer(
1013
config.vision_config,
1014
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
1018
- quant_config=self._maybe_ignore_quant_config(
1019
- self.quant_config),
1015
+ quant_config=self.quant_config,
1016
prefix=maybe_prefix(prefix, "visual"),
1017
use_data_parallel=self.use_data_parallel,
1018
)
@@ -1032,13 +1028,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1028
self.make_empty_intermediate_tensors = (
1029
self.language_model.make_empty_intermediate_tensors)
1030
1035
- def _maybe_ignore_quant_config(self, config: Optional[QuantizationConfig]):
1036
- # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
1037
- # seems to avoid vision encoder sections for some models.
1038
- if isinstance(config, (GPTQConfig, GPTQMarlinConfig)):
1039
- return None
1040
- return config
1041
-
1031
def _validate_and_reshape_mm_tensor(self, mm_input: object,
1032
name: str) -> torch.Tensor:
1033
if not isinstance(mm_input, (torch.Tensor, list)):
vllm/model_executor/models/qwen2_vl.py
+1
-12
@@ -50,9 +50,6 @@ from vllm.model_executor.layers.activation import QuickGELU
50
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
51
RowParallelLinear)
52
from vllm.model_executor.layers.quantization import QuantizationConfig
53
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
54
-from vllm.model_executor.layers.quantization.gptq_marlin import (
55
- GPTQMarlinConfig)
53
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
54
from vllm.model_executor.models.module_mapping import MultiModelKeys
55
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -1270,7 +1267,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1267
self.visual = Qwen2VisionTransformer(
1268
config.vision_config,
1269
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
1273
- quant_config=self._maybe_ignore_quant_config(quant_config),
1270
+ quant_config=quant_config,
1271
prefix=maybe_prefix(prefix, "visual"),
1272
use_data_parallel=self.use_data_parallel,
1273
)
@@ -1286,14 +1283,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1283
self.make_empty_intermediate_tensors = (
1284
self.language_model.make_empty_intermediate_tensors)
1285
1289
- def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
1290
- # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
1291
- # seems to avoid vision encoder sections for some models.
1292
- # See: https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4
1293
- if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
1294
- return None
1295
- return quant_config
1296
-
1286
def _validate_and_reshape_mm_tensor(self, mm_input: object,
1287
name: str) -> torch.Tensor:
1288
if not isinstance(mm_input, (torch.Tensor, list)):
vllm/model_executor/models/qwen3_moe.py
+6
-22
@@ -46,9 +46,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
46
RowParallelLinear)
47
from vllm.model_executor.layers.logits_processor import LogitsProcessor
48
from vllm.model_executor.layers.quantization import QuantizationConfig
49
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
50
-from vllm.model_executor.layers.quantization.gptq_marlin import (
51
- GPTQMarlinConfig)
49
from vllm.model_executor.layers.rotary_embedding import get_rope
50
from vllm.model_executor.layers.vocab_parallel_embedding import (
51
ParallelLMHead, VocabParallelEmbedding)
@@ -149,24 +146,11 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
146
enable_eplb=self.enable_eplb,
147
num_redundant_experts=self.n_redundant_experts)
148
152
- self.gate = ReplicatedLinear(
153
- config.hidden_size,
154
- config.num_experts,
155
- bias=False,
156
- quant_config=self._maybe_ignore_quant_config(quant_config),
157
- prefix=f"{prefix}.gate")
158
-
159
- def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
160
- # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
161
- # seems to avoid gate quantization while AutoRound does.
162
- # See: https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4,
163
- # and https://huggingface.co/jart25/Qwen3-Coder-30B-A3B-Instruct-Int4-gptq
164
- if isinstance(
165
- quant_config,
166
- (GPTQConfig,
167
- GPTQMarlinConfig)) and not quant_config.autoround_version:
168
- return None
169
- return quant_config
149
+ self.gate = ReplicatedLinear(config.hidden_size,
150
+ config.num_experts,
151
+ bias=False,
152
+ quant_config=quant_config,
153
+ prefix=f"{prefix}.gate")
154
155
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
156
assert hidden_states.dim(
@@ -699,4 +683,4 @@ class Qwen3MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA,
683
return loader.load_weights(weights)
684
685
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
702
- return self.model.get_expert_mapping()
\ No newline at end of file
686
+ return self.model.get_expert_mapping()
vllm/model_executor/models/qwen3_next.py
+5
-19
@@ -41,9 +41,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
41
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
42
causal_conv1d_fn, causal_conv1d_update)
43
from vllm.model_executor.layers.quantization import QuantizationConfig
44
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
45
-from vllm.model_executor.layers.quantization.gptq_marlin import (
46
- GPTQMarlinConfig)
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
from vllm.model_executor.layers.vocab_parallel_embedding import (
46
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
@@ -119,12 +116,11 @@ class Qwen3NextSparseMoeBlock(nn.Module):
116
enable_eplb=self.enable_eplb,
117
num_redundant_experts=self.n_redundant_experts)
118
122
- self.gate = ReplicatedLinear(
123
- config.hidden_size,
124
- config.num_experts,
125
- bias=False,
126
- quant_config=self._maybe_ignore_quant_config(quant_config),
127
- prefix=f"{prefix}.gate")
119
+ self.gate = ReplicatedLinear(config.hidden_size,
120
+ config.num_experts,
121
+ bias=False,
122
+ quant_config=quant_config,
123
+ prefix=f"{prefix}.gate")
124
125
if config.shared_expert_intermediate_size > 0:
126
self.shared_expert = Qwen3NextMLP(
@@ -142,16 +138,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
138
1,
139
bias=False)
140
145
- def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
146
- # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
147
- # seems to avoid gate quantization while AutoRound does.
148
- if isinstance(
149
- quant_config,
150
- (GPTQConfig,
151
- GPTQMarlinConfig)) and not quant_config.autoround_version:
152
- return None
153
- return quant_config
154
-
141
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
142
# NOTE: hidden_states can have either 1D or 2D shape.
143
orig_shape = hidden_states.shape
vllm/model_executor/models/qwen3_vl.py
+1
-11
@@ -50,9 +50,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
50
RowParallelLinear)
51
from vllm.model_executor.layers.logits_processor import LogitsProcessor
52
from vllm.model_executor.layers.quantization import QuantizationConfig
53
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
54
-from vllm.model_executor.layers.quantization.gptq_marlin import (
55
- GPTQMarlinConfig)
53
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
54
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
55
from vllm.model_executor.models.module_mapping import MultiModelKeys
@@ -1058,7 +1055,7 @@ class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1055
self.visual = Qwen3_VisionTransformer(
1056
config.vision_config,
1057
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
1061
- quant_config=self._maybe_ignore_quant_config(quant_config),
1058
+ quant_config=quant_config,
1059
prefix=maybe_prefix(prefix, "visual"),
1060
use_data_parallel=self.use_data_parallel,
1061
)
@@ -1116,13 +1113,6 @@ class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1113
for idx in range(self.deepstack_num_level):
1114
self.deepstack_input_embeds[idx][:num_tokens].zero_()
1115
1119
- def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
1120
- # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
1121
- # seems to avoid vision encoder sections for some models.
1122
- if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
1123
- return None
1124
- return quant_config
1125
-
1116
def _validate_and_reshape_mm_tensor(self, mm_input: object,
1117
name: str) -> torch.Tensor:
1118
if not isinstance(mm_input, (torch.Tensor, list)):
vllm/model_executor/models/qwen3_vl_moe.py
+1
-1
@@ -322,7 +322,7 @@ class Qwen3VLMoeForConditionalGeneration(Qwen3VLForConditionalGeneration):
322
self.visual = Qwen3_VisionTransformer(
323
config.vision_config,
324
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
325
- quant_config=self._maybe_ignore_quant_config(quant_config),
325
+ quant_config=quant_config,
326
prefix=maybe_prefix(prefix, "visual"),
327
use_data_parallel=self.use_data_parallel,
328
)
vllm/transformers_utils/config.py
+31
-1
@@ -4,6 +4,7 @@
4
import json
5
import os
6
import time
7
+from dataclasses import asdict
8
from functools import cache, partial
9
from pathlib import Path
10
from typing import Any, Callable, Literal, Optional, TypeVar, Union
@@ -27,7 +28,8 @@ from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
28
from vllm import envs
29
from vllm.logger import init_logger
30
from vllm.transformers_utils.config_parser_base import ConfigParserBase
30
-from vllm.transformers_utils.utils import check_gguf_file
31
+from vllm.transformers_utils.utils import (check_gguf_file,
32
+ parse_safetensors_file_metadata)
33
34
if envs.VLLM_USE_MODELSCOPE:
35
from modelscope import AutoConfig
@@ -999,6 +1001,34 @@ def try_get_tokenizer_config(
1001
return None
1002
1003
1004
+def get_safetensors_params_metadata(
1005
+ model: str,
1006
+ *,
1007
+ revision: Optional[str] = None,
1008
+) -> dict[str, Any]:
1009
+ """
1010
+ Get the safetensors metadata for remote model repository.
1011
+ """
1012
+ full_metadata = {}
1013
+ if (model_path := Path(model)).exists():
1014
+ safetensors_to_check = model_path.glob("*.safetensors")
1015
+ full_metadata = {
1016
+ param_name: info
1017
+ for file_path in safetensors_to_check if file_path.is_file()
1018
+ for param_name, info in parse_safetensors_file_metadata(
1019
+ file_path).items()
1020
+ }
1021
+ else:
1022
+ repo_mt = try_get_safetensors_metadata(model, revision=revision)
1023
+ if repo_mt and (files_mt := repo_mt.files_metadata):
1024
+ full_metadata = {
1025
+ param_name: asdict(info)
1026
+ for file_mt in files_mt.values()
1027
+ for param_name, info in file_mt.tensors.items()
1028
+ }
1029
+ return full_metadata
1030
+
1031
+
1032
def _download_mistral_config_file(model, revision) -> dict:
1033
config_file_name = "params.json"
1034
config_dict = get_hf_file_to_dict(config_file_name, model, revision)
vllm/transformers_utils/utils.py
+10
-1
@@ -2,10 +2,11 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import json
5
+import struct
6
from functools import cache
7
from os import PathLike
8
from pathlib import Path
8
-from typing import Optional, Union
9
+from typing import Any, Optional, Union
10
11
from vllm.envs import VLLM_MODEL_REDIRECT_PATH
12
from vllm.logger import init_logger
@@ -97,3 +98,11 @@ def maybe_model_redirect(model: str) -> str:
98
return redirect_model
99
100
return model
101
+
102
+
103
+def parse_safetensors_file_metadata(
104
+ path: Union[str, PathLike]) -> dict[str, Any]:
105
+ with open(path, "rb") as f:
106
+ length_of_metadata = struct.unpack('<Q', f.read(8))[0]
107
+ metadata = json.loads(f.read(length_of_metadata).decode('utf-8'))
108
+ return metadata