A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Misc][Quark] Upstream Quark format to VLLM (#10765)
Signed-off-by: kewang-xlnx <kewang@xilinx.com> Signed-off-by: kewang2 <kewang2@amd.com> Co-authored-by: kewang2 <kewang2@amd.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
kewang-xlnx committed
Jan 16, 2025 at 00:05 UTC
de0526f668d6918c1884fd3b201308e9049e6be9
32 files changed
+1264
-70
tests/quantization/test_quark.py
new
+30
@@ -0,0 +1,30 @@
1
+"""Test model set-up and weight loading for quark-quantized models.
2
+
3
+Run `pytest tests/quantization/test_quark.py`.
4
+"""
5
+
6
+import torch
7
+
8
+from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
9
+ QuarkLinearMethod, QuarkW8A8Fp8)
10
+
11
+
12
+def test_quark_fp8(vllm_runner):
13
+ model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"
14
+ with vllm_runner(model_path) as llm:
15
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
16
+ layer = model.model.layers[0]
17
+
18
+ qkv_proj = layer.self_attn.qkv_proj
19
+
20
+ assert isinstance(qkv_proj.quant_method, QuarkLinearMethod)
21
+ assert isinstance(qkv_proj.scheme, QuarkW8A8Fp8)
22
+
23
+ if isinstance(qkv_proj.scheme, QuarkW8A8Fp8):
24
+ assert len(qkv_proj.input_scale.shape) == 0
25
+ assert qkv_proj.weight.dtype is torch.float8_e4m3fn
26
+ #assert qkv_proj.weight.dtype is torch.float8_e4m3fnuz
27
+ assert len(qkv_proj.weight_scale.shape) == 0
28
+
29
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
30
+ assert output
vllm/config.py
+1
-1
@@ -553,7 +553,7 @@ class ModelConfig:
553
optimized_quantization_methods = [
554
"fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin",
555
"awq_marlin", "fbgemm_fp8", "compressed_tensors",
556
- "compressed-tensors", "experts_int8"
556
+ "compressed-tensors", "experts_int8", "quark"
557
]
558
if self.quantization is not None:
559
self.quantization = self.quantization.lower()
vllm/model_executor/layers/linear.py
+1
-1
@@ -32,7 +32,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
32
"MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
33
"TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod",
34
"ModelOptFp8LinearMethod", "IPEXAWQLinearMethod", "IPEXGPTQLinearMethod",
35
- "HQQMarlinMethod"
35
+ "HQQMarlinMethod", "QuarkLinearMethod"
36
]
37
38
vllm/model_executor/layers/quantization/__init__.py
+4
@@ -26,6 +26,7 @@ QUANTIZATION_METHODS: List[str] = [
26
"experts_int8",
27
"neuron_quant",
28
"ipex",
29
+ "quark"
30
]
31
32
@@ -34,6 +35,8 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
35
raise ValueError(f"Invalid quantization method: {quantization}")
36
37
# lazy import to avoid triggering `torch.compile` too early
38
+ from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
39
+
40
from .aqlm import AQLMConfig
41
from .awq import AWQConfig
42
from .awq_marlin import AWQMarlinConfig
@@ -79,6 +82,7 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
82
"experts_int8": ExpertsInt8Config,
83
"neuron_quant": NeuronQuantConfig,
84
"ipex": IPEXConfig,
85
+ "quark": QuarkConfig
86
}
87
88
return method_to_config[quantization]
vllm/model_executor/layers/quantization/base_config.py
+3
@@ -133,3 +133,6 @@ class QuantizationConfig(ABC):
133
method.
134
"""
135
raise NotImplementedError
136
+
137
+ def get_cache_scale(self, name: str) -> Optional[str]:
138
+ return None
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+16
@@ -412,6 +412,22 @@ class CompressedTensorsConfig(QuantizationConfig):
412
self._check_scheme_supported(scheme.get_min_capability())
413
return scheme
414
415
+ def get_cache_scale(self, name: str) -> Optional[str]:
416
+ """
417
+ Check whether the param name matches the format for k/v cache scales
418
+ in compressed-tensors. If this is the case, return its equivalent
419
+ param name expected by vLLM
420
+
421
+ :param name: param name
422
+ :return: matching param name for KV cache scale in vLLM
423
+ """
424
+ if name.endswith(".output_scale") and ".k_proj" in name:
425
+ return name.replace(".k_proj.output_scale", ".attn.k_scale")
426
+ if name.endswith(".output_scale") and ".v_proj" in name:
427
+ return name.replace(".v_proj.output_scale", ".attn.v_scale")
428
+ # If no matches, return None
429
+ return None
430
+
431
@staticmethod
432
def supports_cutlass_24(
433
weight_quant: Optional[QuantizationArgs],
vllm/model_executor/layers/quantization/compressed_tensors/triton_scaled_mm.py
+4
@@ -136,6 +136,10 @@ def triton_scaled_mm(input: torch.Tensor,
136
assert N > 0 and K > 0 and M > 0
137
assert weight.shape[0] == K
138
assert input.dtype == weight.dtype
139
+
140
+ scale_a = scale_a.reshape(-1, 1) if scale_a.dim() <= 1 else scale_a
141
+ scale_b = scale_b.reshape(-1, 1) if scale_b.dim() <= 1 else scale_b
142
+
143
assert scale_a.dtype == scale_b.dtype and scale_a.is_floating_point()
144
assert scale_a.shape == torch.Size([1, 1]) or scale_a.shape == torch.Size(
145
[M, 1])
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
-17
@@ -133,23 +133,6 @@ def _find_first_match(value: str,
133
return None
134
135
136
-def get_compressed_tensors_cache_scale(name: str) -> Optional[str]:
137
- """
138
- Check whether the param name matches the format for k/v cache scales
139
- in compressed-tensors. If this is the case, return its equivalent
140
- param name expected by vLLM
141
-
142
- :param name: param name
143
- :return: matching param name for KV cache scale in vLLM
144
- """
145
- if name.endswith(".output_scale") and ".k_proj" in name:
146
- return name.replace(".k_proj.output_scale", ".attn.k_scale")
147
- if name.endswith(".output_scale") and ".v_proj" in name:
148
- return name.replace(".v_proj.output_scale", ".attn.v_scale")
149
- # If no matches, return None
150
- return None
151
-
152
-
136
def _is_equal_or_regex_match(value: str,
137
target: str,
138
check_contains: bool = False) -> bool:
vllm/model_executor/layers/quantization/quark/__init__.py
vllm/model_executor/layers/quantization/quark/quark.py
new
+387
@@ -0,0 +1,387 @@
1
+import fnmatch
2
+import re
3
+from typing import Any, Dict, List, Optional, cast
4
+
5
+import torch
6
+
7
+from vllm.model_executor.layers.fused_moe import FusedMoE
8
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9
+ UnquantizedLinearMethod)
10
+from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
11
+ QuantizationConfig, QuantizeMethodBase)
12
+from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
13
+from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
14
+ QuarkMoEMethod)
15
+from vllm.model_executor.layers.quantization.quark.schemes import (
16
+ QuarkScheme, QuarkW8A8Fp8, QuarkW8A8Int8)
17
+from vllm.model_executor.layers.quantization.quark.utils import (
18
+ deep_compare, should_ignore_layer)
19
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
20
+ FUSED_LAYER_NAME_MAPPING)
21
+from vllm.platforms import current_platform
22
+
23
+__all__ = ["QuarkLinearMethod"]
24
+
25
+
26
+class QuarkConfig(QuantizationConfig):
27
+
28
+ def __init__(self,
29
+ quant_config: Dict[str, Any],
30
+ kv_cache_group: Optional[List[str]] = None,
31
+ kv_cache_config: Optional[Dict[str, Any]] = None,
32
+ pack_method: str = "reorder"):
33
+ if kv_cache_group is None:
34
+ kv_cache_group = []
35
+ self.quant_config = quant_config
36
+ self.kv_cache_group = kv_cache_group
37
+ self.kv_cache_config = kv_cache_config
38
+ self.pack_method = pack_method
39
+
40
+ def get_linear_method(self) -> "QuarkLinearMethod":
41
+ return QuarkLinearMethod(self)
42
+
43
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
44
+ return [torch.float16, torch.bfloat16]
45
+
46
+ @classmethod
47
+ def get_min_capability(cls) -> int:
48
+ return 70
49
+
50
+ def get_name(self) -> str:
51
+ return "quark"
52
+
53
+ def get_quant_method(self, layer: torch.nn.Module,
54
+ prefix: str) -> Optional["QuantizeMethodBase"]:
55
+ from vllm.attention.layer import Attention # Avoid circular import
56
+
57
+ # Check if the layer is skipped for quantization.
58
+ exclude_layers = cast(List[str], self.quant_config.get("exclude"))
59
+ if should_ignore_layer(prefix, ignore=exclude_layers):
60
+ return UnquantizedLinearMethod()
61
+ if isinstance(layer, LinearBase):
62
+ scheme = self.get_scheme(layer=layer, layer_name=prefix)
63
+ layer.scheme = scheme
64
+ return QuarkLinearMethod(self)
65
+ if isinstance(layer, Attention):
66
+ return QuarkKVCacheMethod(self)
67
+ if isinstance(layer, FusedMoE):
68
+ return QuarkMoEMethod.get_moe_method(self,
69
+ module=layer,
70
+ layer_name=prefix)
71
+ return None
72
+
73
+ @classmethod
74
+ def from_config(cls, config: Dict[str, Any]) -> "QuarkConfig":
75
+ export_config = config.get("export")
76
+ if export_config is None:
77
+ raise ValueError("The export key should be included in "
78
+ "the configurations of Quark quantized model")
79
+ kv_cache_group = cast(List[str], export_config.get("kv_cache_group"))
80
+ pack_method = cast(str, export_config.get("pack_method"))
81
+
82
+ # In the export model of quark, the quantization configuration
83
+ # of kv_cache is stored in layer_quant_config. First, it is
84
+ # judged whether kv_cache_group exists, and then it is judged
85
+ # whether layer_quant_config has a quantization configuration
86
+ # that matches kv_cache.
87
+ if len(kv_cache_group) == 0:
88
+ kv_cache_config = None
89
+ else:
90
+ kv_cache_set = set(kv_cache_group)
91
+ layer_quant_config = cast(Dict[str, Any],
92
+ config.get("layer_quant_config"))
93
+ layer_quant_names = list(layer_quant_config.keys())
94
+ layer_quant_set = set(layer_quant_names)
95
+
96
+ if not kv_cache_set.issubset(layer_quant_set):
97
+ raise ValueError("The Quark quantized model has the "
98
+ "kv_cache_group parameter setting, "
99
+ "but no kv_cache quantization settings "
100
+ "were found in the quantization "
101
+ "configuration.")
102
+
103
+ q_configs = [
104
+ cast(Dict[str, Any], layer_quant_config.get(name))
105
+ for name in kv_cache_group
106
+ ]
107
+ if not all(
108
+ deep_compare(q_config, q_configs[0])
109
+ for q_config in q_configs):
110
+ raise ValueError(
111
+ "The quantization method used for kv_cache should "
112
+ "be the same, but the quantization method for the "
113
+ "kv_cache layer in the config is different.")
114
+ kv_cache_config = q_configs[0].get("output_tensors")
115
+ if kv_cache_config is None:
116
+ raise ValueError(
117
+ "The kv_cache quantization configuration is empty.")
118
+
119
+ # Since we have already set kv_cache quantization configurations,
120
+ # we will remove the quantization configuration for the
121
+ # output_tensors corresponding to the kv_cache layer.
122
+ for q_config in q_configs:
123
+ q_config["output_tensors"] = None
124
+
125
+ return cls(quant_config=config,
126
+ kv_cache_group=kv_cache_group,
127
+ kv_cache_config=kv_cache_config,
128
+ pack_method=pack_method)
129
+
130
+ @classmethod
131
+ def get_config_filenames(cls) -> List[str]:
132
+ return []
133
+
134
+ def _check_scheme_supported(self,
135
+ min_capability: int,
136
+ error: bool = True) -> bool:
137
+ capability_tuple = current_platform.get_device_capability()
138
+
139
+ if capability_tuple is not None:
140
+ capability = capability_tuple.to_int()
141
+ supported = capability >= min_capability
142
+ if error and not supported:
143
+ raise RuntimeError(
144
+ "Quantization scheme is not supported for ",
145
+ f"the current GPU. Min capability: {min_capability}. ",
146
+ f"Current capability: {capability}.")
147
+ return supported
148
+ else:
149
+ return False
150
+
151
+ def _is_fp8_w8a8(self, weight_quant: Optional[Dict[str, Any]],
152
+ input_quant: Optional[Dict[str, Any]]) -> bool:
153
+ # Confirm weights and input quantized.
154
+ if weight_quant is None or input_quant is None:
155
+ return False
156
+
157
+ # Confirm weight scheme is supported
158
+ is_fp8_dtype = (weight_quant.get("dtype") == "fp8_e4m3"
159
+ and input_quant.get("dtype") == "fp8_e4m3")
160
+ is_static_weight = not weight_quant.get("is_dynamic")
161
+ is_per_tensor_or_channel_weight = (weight_quant.get("qscheme")
162
+ in ["per_tensor", "per_channel"])
163
+
164
+ if not (is_fp8_dtype and is_static_weight
165
+ and is_per_tensor_or_channel_weight):
166
+ return False
167
+
168
+ # Dynamic quantization is always supported if weights supported.
169
+ if input_quant.get("is_dynamic"):
170
+ return True
171
+
172
+ # Confirm activation scheme is supported.
173
+ is_per_tensor_activation = (input_quant.get("qscheme") == "per_tensor")
174
+ return is_per_tensor_activation
175
+
176
+ def _is_static_tensor_w8a8(self, weight_quant: Optional[Dict[str, Any]],
177
+ input_quant: Optional[Dict[str, Any]]) -> bool:
178
+ # Confirm weights and input quantized.
179
+ if weight_quant is None or input_quant is None:
180
+ return False
181
+
182
+ is_int8_dtype = (weight_quant.get("dtype") == "int8"
183
+ and input_quant.get("dtype") == "int8")
184
+
185
+ is_tensor = (weight_quant.get("qscheme")
186
+ in ["per_tensor", "per_channel"]
187
+ and input_quant.get("qscheme") == "per_tensor")
188
+
189
+ is_static = (not weight_quant.get("is_dynamic")
190
+ and not input_quant.get("is_dynamic"))
191
+
192
+ is_weight_symmetric = (weight_quant.get("symmetric") is True)
193
+
194
+ # Both symmetric and asymmetric input quantization supported.
195
+ # Only symmetric weight quantization supported.
196
+ return is_int8_dtype and is_tensor and is_weight_symmetric and is_static
197
+
198
+ def _find_matched_config(self, layer_name: str,
199
+ module: torch.nn.Module) -> Dict[str, Any]:
200
+
201
+ proj_name = layer_name.split(".")[-1]
202
+ if proj_name in FUSED_LAYER_NAME_MAPPING:
203
+ shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
204
+
205
+ # Convert fused_name --> [shard_names]
206
+ shard_names = [
207
+ layer_name.replace(proj_name, shard_proj_name)
208
+ for shard_proj_name in shard_proj_names
209
+ ]
210
+ shard_configs = [
211
+ self._find_matched_config(shard_name, module)
212
+ for shard_name in shard_names
213
+ ]
214
+ if not all(
215
+ deep_compare(q_config, shard_configs[0])
216
+ for q_config in shard_configs):
217
+ raise ValueError(
218
+ f"Found a different quantization configuration for "
219
+ f"{shard_proj_names} in {layer_name}. vLLM "
220
+ "requires all to use the same scheme.")
221
+ return shard_configs[0]
222
+ else:
223
+ layer_quant_config = cast(
224
+ Dict[str, Any], self.quant_config.get("layer_quant_config"))
225
+ for name_pattern in layer_quant_config:
226
+ if fnmatch.fnmatch(layer_name, name_pattern):
227
+ return layer_quant_config[name_pattern]
228
+
229
+ layer_type = cast(str, type(module))
230
+ layer_type_quant_config = cast(
231
+ Dict[str, Any],
232
+ self.quant_config.get("layer_type_quant_config"))
233
+ if layer_type in layer_type_quant_config:
234
+ return layer_type_quant_config[layer_type]
235
+
236
+ global_quant_config = cast(
237
+ Dict[str, Any], self.quant_config.get("global_quant_config"))
238
+ return global_quant_config
239
+
240
+ def _get_scheme_from_config(self, config: Dict[str, Any]) -> "QuarkScheme":
241
+ if config.get("output_tensors") or config.get("bias"):
242
+ raise NotImplementedError(
243
+ "Currently, Quark models with output_tensors "
244
+ "and bias quantized are not supported")
245
+ weight_config = cast(Dict[str, Any], config.get("weight"))
246
+ input_config = cast(Dict[str, Any], config.get("input_tensors"))
247
+
248
+ if self._is_fp8_w8a8(weight_config, input_config):
249
+ is_fp8_w8a8_supported = self._check_scheme_supported(
250
+ QuarkW8A8Fp8.get_min_capability(), error=False)
251
+ if is_fp8_w8a8_supported:
252
+ weight_qscheme = cast(str, weight_config.get("qscheme"))
253
+ input_static = (input_config is not None and
254
+ not cast(bool, input_config.get("is_dynamic")))
255
+ return QuarkW8A8Fp8(qscheme=weight_qscheme,
256
+ is_static_input_scheme=input_static)
257
+ elif self._is_static_tensor_w8a8(weight_config, input_config):
258
+ weight_qscheme = cast(str, weight_config.get("qscheme"))
259
+ return QuarkW8A8Int8(qscheme=weight_qscheme,
260
+ is_static_input_scheme=True,
261
+ input_symmetric=input_config.get("symmetric"))
262
+
263
+ raise NotImplementedError("No quark compatible scheme was found. "
264
+ f"Weight config: {weight_config}, "
265
+ f"Input config: {input_config}")
266
+
267
+ def get_scheme(self, layer: torch.nn.Module,
268
+ layer_name: str) -> "QuarkScheme":
269
+
270
+ layer_quant_config = self._find_matched_config(layer_name, layer)
271
+
272
+ # Find the quant_scheme
273
+ scheme = self._get_scheme_from_config(layer_quant_config)
274
+ # Raise error if device does not support the scheme
275
+ # (e.g. fp8 needs ada lovelace)
276
+ self._check_scheme_supported(scheme.get_min_capability())
277
+
278
+ return scheme
279
+
280
+ def get_cache_scale(self, name: str) -> Optional[str]:
281
+ """
282
+ Check whether the param name matches the format for k/v cache scales
283
+ in quark. If this is the case, return its equivalent param name
284
+ expected by vLLM
285
+
286
+ :param name: param name
287
+ :return: matching param name for KV cache scale in vLLM
288
+ """
289
+ if self.kv_cache_group is None or len(self.kv_cache_group) == 0:
290
+ return None
291
+
292
+ kv_proj_names = [
293
+ re.split(r"[*.]", kv_cache)[-1] for kv_cache in self.kv_cache_group
294
+ ]
295
+ if name.endswith(".output_scale"):
296
+ if len(kv_proj_names) == 1 and kv_proj_names[0] in name:
297
+ kv_output_scale_name = "." + kv_proj_names[0] + ".output_scale"
298
+ return name.replace(kv_output_scale_name, ".attn.k_scale")
299
+
300
+ elif len(kv_proj_names) == 2:
301
+ for kv_proj_name in kv_proj_names:
302
+ if kv_proj_name in name and kv_proj_name == "k_proj":
303
+ return name.replace(".k_proj.output_scale",
304
+ ".attn.k_scale")
305
+ elif kv_proj_name in name and kv_proj_name == "v_proj":
306
+ return name.replace(".v_proj.output_scale",
307
+ ".attn.v_scale")
308
+
309
+ # If no matches, return None
310
+ return None
311
+
312
+
313
+class QuarkLinearMethod(LinearMethodBase):
314
+
315
+ def __init__(self, quantization_config: QuarkConfig):
316
+ self.quantization_config = quantization_config
317
+
318
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
319
+ layer.scheme.process_weights_after_loading(layer)
320
+
321
+ def create_weights(self, layer: torch.nn.Module,
322
+ input_size_per_partition: int,
323
+ output_partition_sizes: List[int], input_size: int,
324
+ output_size: int, params_dtype: torch.dtype,
325
+ **extra_weight_attrs):
326
+ """
327
+ Use the CompressedTensorsScheme associated with each layer to create
328
+ the necessary parameters for the layer. See LinearMethodBase for param
329
+ details
330
+ """
331
+ weight_loader = extra_weight_attrs.get("weight_loader")
332
+ layer.scheme.create_weights(
333
+ layer=layer,
334
+ input_size=input_size,
335
+ input_size_per_partition=input_size_per_partition,
336
+ output_partition_sizes=output_partition_sizes,
337
+ output_size=output_size,
338
+ params_dtype=params_dtype,
339
+ weight_loader=weight_loader)
340
+
341
+ def apply(self,
342
+ layer: torch.nn.Module,
343
+ x: torch.Tensor,
344
+ bias: Optional[torch.Tensor] = None):
345
+ """
346
+ Use the output of create_weights and the CompressedTensorsScheme
347
+ associated with the layer to apply the forward pass with the
348
+ layer input. See LinearMethodBase for param details
349
+
350
+ """
351
+ scheme = layer.scheme
352
+ if scheme is None:
353
+ raise ValueError("A scheme must be defined for each layer")
354
+ return scheme.apply_weights(layer, x, bias=bias)
355
+
356
+
357
+class QuarkKVCacheMethod(BaseKVCacheMethod):
358
+ """
359
+ Supports loading kv-cache scaling factors from quark checkpoints.
360
+ """
361
+
362
+ def __init__(self, quant_config: QuarkConfig):
363
+ self.validate_kv_cache_config(quant_config.kv_cache_config)
364
+ super().__init__(quant_config)
365
+
366
+ @staticmethod
367
+ def validate_kv_cache_config(kv_cache_config: Optional[Dict[str, Any]]):
368
+ """
369
+ Validator for the kv cache configuration. Useful for controlling the
370
+ kv cache quantization schemes, that are being supported in vLLM
371
+ :param kv_cache_config: the quark kv cache scheme
372
+ """
373
+ if kv_cache_config is None:
374
+ return
375
+
376
+ dtype = kv_cache_config.get("dtype")
377
+ if dtype != "fp8_e4m3":
378
+ raise NotImplementedError(
379
+ "Currently supported kv cache quantization is "
380
+ f"dtype=fp8_e4m3, however received {dtype}")
381
+
382
+ qscheme = kv_cache_config.get("qscheme")
383
+ if qscheme != "per_tensor":
384
+ raise NotImplementedError(
385
+ "Only support per-tensor scaling factor "
386
+ "for quark KV cache. "
387
+ f"Expected qscheme: per_tensor, found qscheme: {qscheme}")
vllm/model_executor/layers/quantization/quark/quark_moe.py
new
+225
@@ -0,0 +1,225 @@
1
+from typing import Any, Callable, Dict, Optional
2
+
3
+import torch
4
+
5
+import vllm.model_executor.layers.fused_moe # noqa
6
+from vllm import _custom_ops as ops
7
+from vllm.logger import init_logger
8
+from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
9
+ FusedMoeWeightScaleSupported)
10
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11
+ all_close_1d, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize)
12
+from vllm.model_executor.utils import set_weight_attrs
13
+from vllm.platforms import current_platform
14
+
15
+logger = init_logger(__name__)
16
+
17
+__all__ = ["QuarkMoEMethod", "QuarkW8A8Fp8MoEMethod"]
18
+
19
+
20
+class QuarkMoEMethod(FusedMoEMethodBase):
21
+
22
+ @staticmethod
23
+ def get_moe_method(
24
+ quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821
25
+ module: torch.nn.Module,
26
+ layer_name: str) -> "QuarkMoEMethod":
27
+ layer_quant_config = quant_config._find_matched_config(
28
+ layer_name, module)
29
+
30
+ if (layer_quant_config.get("output_tensors")
31
+ or layer_quant_config.get("bias")):
32
+ raise NotImplementedError("Currently, Quark models with "
33
+ "output_tensors and bias "
34
+ "quantized are not supported")
35
+ weight_config = layer_quant_config.get("weight")
36
+ input_config = layer_quant_config.get("input_tensors")
37
+
38
+ if quant_config._is_fp8_w8a8(weight_config, input_config):
39
+ return QuarkW8A8Fp8MoEMethod(weight_config, input_config)
40
+ else:
41
+ raise RuntimeError("Unsupported FusedMoe scheme")
42
+
43
+
44
+class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
45
+
46
+ def __init__(self, weight_config: Dict[str, Any], input_config: Dict[str,
47
+ Any]):
48
+ self.weight_quant = weight_config
49
+ self.input_quant = input_config
50
+
51
+ weight_qscheme = self.weight_quant.get("qscheme")
52
+ input_qscheme = self.input_quant.get("qscheme")
53
+ if not (weight_qscheme == "per_tensor"
54
+ and input_qscheme == "per_tensor"):
55
+ raise ValueError(
56
+ "For FP8 Fused MoE layers, only per-tensor scales"
57
+ "for weights and activations are supported. Found "
58
+ f"{weight_qscheme}, {input_qscheme}") # noqa E501
59
+
60
+ self.static_input_scales = not self.input_quant.get("is_dynamic")
61
+
62
+ def create_weights(self, layer: torch.nn.Module, num_experts: int,
63
+ hidden_size: int, intermediate_size: int,
64
+ params_dtype: torch.dtype, **extra_weight_attrs):
65
+
66
+ params_dtype = torch.float8_e4m3fn
67
+
68
+ # WEIGHTS
69
+ w13_weight = torch.nn.Parameter(torch.empty(num_experts,
70
+ 2 * intermediate_size,
71
+ hidden_size,
72
+ dtype=params_dtype),
73
+ requires_grad=False)
74
+ layer.register_parameter("w13_weight", w13_weight)
75
+ set_weight_attrs(w13_weight, extra_weight_attrs)
76
+
77
+ w2_weight = torch.nn.Parameter(torch.empty(num_experts,
78
+ hidden_size,
79
+ intermediate_size,
80
+ dtype=params_dtype),
81
+ requires_grad=False)
82
+ layer.register_parameter("w2_weight", w2_weight)
83
+ set_weight_attrs(w2_weight, extra_weight_attrs)
84
+
85
+ # WEIGHT_SCALES
86
+ # Allocate 2 scales for w1 and w3 respectively.
87
+ # They will be combined to a single scale after weight loading.
88
+ w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
89
+ 2,
90
+ dtype=torch.float32),
91
+ requires_grad=False)
92
+ layer.register_parameter("w13_weight_scale", w13_weight_scale)
93
+
94
+ w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
95
+ dtype=torch.float32),
96
+ requires_grad=False)
97
+ layer.register_parameter("w2_weight_scale", w2_weight_scale)
98
+ # Add the quantization method used (per tensor/grouped/channel)
99
+ # to ensure the weight scales are loaded in properly
100
+ extra_weight_attrs.update(
101
+ {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value})
102
+ set_weight_attrs(w13_weight_scale, extra_weight_attrs)
103
+ set_weight_attrs(w2_weight_scale, extra_weight_attrs)
104
+
105
+ # INPUT_SCALES
106
+ if self.static_input_scales:
107
+ w13_input_scale = torch.nn.Parameter(torch.ones(
108
+ num_experts, dtype=torch.float32),
109
+ requires_grad=False)
110
+ layer.register_parameter("w13_input_scale", w13_input_scale)
111
+ set_weight_attrs(w13_input_scale, extra_weight_attrs)
112
+
113
+ w2_input_scale = torch.nn.Parameter(torch.ones(
114
+ num_experts, dtype=torch.float32),
115
+ requires_grad=False)
116
+ layer.register_parameter("w2_input_scale", w2_input_scale)
117
+ set_weight_attrs(w2_input_scale, extra_weight_attrs)
118
+ else:
119
+ layer.w13_input_scale = None
120
+ layer.w2_input_scale = None
121
+
122
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
123
+ # Fp8 moe kernels require a single activation scale.
124
+ # We take the max of all the scales in case they differ.
125
+ if self.static_input_scales:
126
+ if (layer.w13_input_scale is None or layer.w2_input_scale is None):
127
+ raise ValueError(
128
+ "QuantConfig has static quantization, but found "
129
+ "activation scales are None.")
130
+ if (not all_close_1d(layer.w13_input_scale)
131
+ or not all_close_1d(layer.w2_input_scale)):
132
+ logger.warning_once(
133
+ "Found input_scales that are not equal for "
134
+ "fp8 MoE layer. Using the maximum across experts "
135
+ "for each layer. ")
136
+ layer.w13_input_scale = torch.nn.Parameter(
137
+ layer.w13_input_scale.max(), requires_grad=False)
138
+ layer.w2_input_scale = torch.nn.Parameter(
139
+ layer.w2_input_scale.max(), requires_grad=False)
140
+
141
+ # If rocm, normalize the weights and scales to e4m3fnuz
142
+ if current_platform.is_rocm():
143
+ # Normalize the weights and scales
144
+ w13_weight, w13_weight_scale, w13_input_scale = \
145
+ normalize_e4m3fn_to_e4m3fnuz(
146
+ layer.w13_weight, layer.w13_weight_scale,
147
+ layer.w13_input_scale)
148
+ w2_weight, w2_weight_scale, w2_input_scale = \
149
+ normalize_e4m3fn_to_e4m3fnuz(
150
+ layer.w2_weight, layer.w2_weight_scale,
151
+ layer.w2_input_scale)
152
+ # Reset the parameter
153
+ layer.w13_weight = torch.nn.Parameter(w13_weight,
154
+ requires_grad=False)
155
+ layer.w13_weight_scale = torch.nn.Parameter(w13_weight_scale,
156
+ requires_grad=False)
157
+ if w13_input_scale is not None:
158
+ layer.w13_input_scale = torch.nn.Parameter(w13_input_scale,
159
+ requires_grad=False)
160
+ layer.w2_weight = torch.nn.Parameter(w2_weight,
161
+ requires_grad=False)
162
+ layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale,
163
+ requires_grad=False)
164
+ if w2_input_scale is not None:
165
+ layer.w2_input_scale = torch.nn.Parameter(w2_input_scale,
166
+ requires_grad=False)
167
+
168
+ # Fp8 moe kernel needs single weight scale for w13 per expert.
169
+ # We take the max then dequant and requant each expert.
170
+ assert layer.w13_weight_scale is not None
171
+ shard_size = layer.intermediate_size_per_partition
172
+ max_w13_scales = layer.w13_weight_scale.max(dim=1).values
173
+ for expert_id in range(layer.num_experts):
174
+ start = 0
175
+ for shard_id in range(2):
176
+ dq_weight = per_tensor_dequantize(
177
+ layer.w13_weight[expert_id][start:start + shard_size, :],
178
+ layer.w13_weight_scale[expert_id][shard_id])
179
+ layer.w13_weight[expert_id][
180
+ start:start + shard_size, :], _ = ops.scaled_fp8_quant(
181
+ dq_weight, max_w13_scales[expert_id])
182
+ start += shard_size
183
+
184
+ layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales,
185
+ requires_grad=False)
186
+
187
+ def apply(
188
+ self,
189
+ layer: torch.nn.Module,
190
+ x: torch.Tensor,
191
+ router_logits: torch.Tensor,
192
+ top_k: int,
193
+ renormalize: bool,
194
+ use_grouped_topk: bool = False,
195
+ topk_group: Optional[int] = None,
196
+ num_expert_group: Optional[int] = None,
197
+ custom_routing_function: Optional[Callable] = None,
198
+ scoring_func: str = "softmax",
199
+ e_score_correction_bias: Optional[torch.Tensor] = None,
200
+ ) -> torch.Tensor:
201
+ from vllm.model_executor.layers.fused_moe import fused_experts
202
+
203
+ topk_weights, topk_ids = FusedMoE.select_experts(
204
+ hidden_states=x,
205
+ router_logits=router_logits,
206
+ use_grouped_topk=use_grouped_topk,
207
+ top_k=top_k,
208
+ renormalize=renormalize,
209
+ topk_group=topk_group,
210
+ num_expert_group=num_expert_group,
211
+ custom_routing_function=custom_routing_function,
212
+ scoring_func=scoring_func,
213
+ e_score_correction_bias=e_score_correction_bias)
214
+
215
+ return fused_experts(x,
216
+ layer.w13_weight,
217
+ layer.w2_weight,
218
+ topk_weights=topk_weights,
219
+ topk_ids=topk_ids,
220
+ inplace=True,
221
+ use_fp8_w8a8=True,
222
+ w1_scale=layer.w13_weight_scale,
223
+ w2_scale=layer.w2_weight_scale,
224
+ a1_scale=layer.w13_input_scale,
225
+ a2_scale=layer.w2_input_scale)
vllm/model_executor/layers/quantization/quark/schemes/__init__.py
new
+5
@@ -0,0 +1,5 @@
1
+from .quark_scheme import QuarkScheme
2
+from .quark_w8a8_fp8 import QuarkW8A8Fp8
3
+from .quark_w8a8_int8 import QuarkW8A8Int8
4
+
5
+__all__ = ["QuarkScheme", "QuarkW8A8Fp8", "QuarkW8A8Int8"]
vllm/model_executor/layers/quantization/quark/schemes/quark_scheme.py
new
+52
@@ -0,0 +1,52 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Optional
3
+
4
+import torch
5
+
6
+__all__ = ["QuarkScheme"]
7
+
8
+
9
+class QuarkScheme(ABC):
10
+ """
11
+ Abstract class used to describe the weight creation and forward pass
12
+ of different quantization schemes supported by Quark.
13
+ """
14
+
15
+ @classmethod
16
+ @abstractmethod
17
+ def get_min_capability(cls) -> int:
18
+ """
19
+ Get minimum device capability.
20
+ """
21
+ raise NotImplementedError
22
+
23
+ @abstractmethod
24
+ def create_weights(self, *args, **kwargs):
25
+ """
26
+ Weight creation for the particular scheme. Inputs to this function
27
+
28
+ """
29
+ raise NotImplementedError
30
+
31
+ @abstractmethod
32
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
33
+ bias: Optional[torch.Tensor]):
34
+ """
35
+ Run the forward pass for the particular scheme. This is where
36
+ scheme-specific dequant/quant steps/kernels should be applied.
37
+
38
+ :param layer: torch.nn.Module with the registered weights and
39
+ other parameters relevant to the particular scheme.
40
+ :param x: input to the layer
41
+ :param bias: bias parameter
42
+
43
+ """
44
+ raise NotImplementedError
45
+
46
+ @abstractmethod
47
+ def process_weights_after_loading(self, layer: torch.nn.Module):
48
+ """
49
+ Called after weight loading is complete for any cleanup that
50
+ needs to occur.
51
+ """
52
+ raise NotImplementedError
vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py
new
+140
@@ -0,0 +1,140 @@
1
+from typing import Callable, List, Optional
2
+
3
+import torch
4
+from torch.nn import Parameter
5
+
6
+from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
7
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
8
+ apply_fp8_linear, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz,
9
+ requantize_with_max_scale)
10
+from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
11
+ ModelWeightParameter,
12
+ PerTensorScaleParameter)
13
+from vllm.platforms import current_platform
14
+
15
+__all__ = ["QuarkW8A8Fp8"]
16
+
17
+
18
+class QuarkW8A8Fp8(QuarkScheme):
19
+
20
+ def __init__(self, qscheme: str, is_static_input_scheme: Optional[bool]):
21
+ self.qscheme = qscheme
22
+ self.is_static_input_scheme = is_static_input_scheme
23
+ self.cutlass_fp8_supported = cutlass_fp8_supported()
24
+
25
+ @classmethod
26
+ def get_min_capability(cls) -> int:
27
+ # lovelace and up
28
+ return 89
29
+
30
+ def process_weights_after_loading(self, layer) -> None:
31
+ # If per tensor, when we have a fused module (e.g. QKV) with per
32
+ # tensor scales (thus N scales being passed to the kernel),
33
+ # requantize so we can always run per tensor
34
+ if self.qscheme == "per_tensor":
35
+ max_w_scale, weight = requantize_with_max_scale(
36
+ weight=layer.weight,
37
+ weight_scale=layer.weight_scale,
38
+ logical_widths=layer.logical_widths,
39
+ )
40
+
41
+ if current_platform.is_rocm():
42
+ weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
43
+ weight=weight,
44
+ weight_scale=max_w_scale,
45
+ input_scale=layer.input_scale)
46
+ if input_scale is not None:
47
+ layer.input_scale = Parameter(input_scale,
48
+ requires_grad=False)
49
+
50
+ layer.weight = Parameter(weight.t(), requires_grad=False)
51
+ layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
52
+
53
+ # If channelwise, scales are already lined up, so just transpose.
54
+ elif self.qscheme == "per_channel":
55
+ weight = layer.weight
56
+
57
+ if current_platform.is_rocm():
58
+ weight, weight_scale, input_scale = \
59
+ normalize_e4m3fn_to_e4m3fnuz(
60
+ weight=weight,
61
+ weight_scale=layer.weight_scale,
62
+ input_scale=layer.input_scale)
63
+ if input_scale is not None:
64
+ layer.input_scale = Parameter(input_scale,
65
+ requires_grad=False)
66
+ else:
67
+ weight_scale = layer.weight_scale.data
68
+
69
+ layer.weight = Parameter(weight.t(), requires_grad=False)
70
+ # required by torch.compile to be torch.nn.Parameter
71
+ layer.weight_scale = Parameter(weight_scale, requires_grad=False)
72
+
73
+ else:
74
+ raise ValueError(f"Unknown quantization scheme {self.qscheme}")
75
+
76
+ # INPUT SCALE
77
+ if self.is_static_input_scheme:
78
+ layer.input_scale = Parameter(layer.input_scale.max(),
79
+ requires_grad=False)
80
+ else:
81
+ layer.input_scale = None
82
+
83
+ def create_weights(self, layer: torch.nn.Module,
84
+ output_partition_sizes: List[int],
85
+ input_size_per_partition: int,
86
+ params_dtype: torch.dtype, weight_loader: Callable,
87
+ **kwargs):
88
+ output_size_per_partition = sum(output_partition_sizes)
89
+ layer.logical_widths = output_partition_sizes
90
+
91
+ # WEIGHT
92
+ weight = ModelWeightParameter(data=torch.empty(
93
+ output_size_per_partition,
94
+ input_size_per_partition,
95
+ dtype=torch.float8_e4m3fn),
96
+ input_dim=1,
97
+ output_dim=0,
98
+ weight_loader=weight_loader)
99
+ layer.register_parameter("weight", weight)
100
+
101
+ # WEIGHT SCALE
102
+ # TODO: update create_xxx_parameter functions to return
103
+ # the newly added parameters
104
+ if self.qscheme == "per_channel":
105
+ weight_scale = ChannelQuantScaleParameter(
106
+ data=torch.empty((sum(output_partition_sizes), 1),
107
+ dtype=torch.float32),
108
+ output_dim=0,
109
+ weight_loader=weight_loader)
110
+ else:
111
+ assert self.qscheme == "per_tensor"
112
+ weight_scale = PerTensorScaleParameter(data=torch.empty(
113
+ len(output_partition_sizes), dtype=torch.float32),
114
+ weight_loader=weight_loader)
115
+
116
+ # min requirement for fp8 kernels
117
+ weight_scale[:] = torch.finfo(torch.float32).min
118
+ layer.register_parameter("weight_scale", weight_scale)
119
+
120
+ # INPUT SCALE
121
+ if self.is_static_input_scheme:
122
+ input_scale = PerTensorScaleParameter(data=torch.empty(
123
+ len(output_partition_sizes), dtype=torch.float32),
124
+ weight_loader=weight_loader)
125
+ input_scale[:] = torch.finfo(torch.float32).min
126
+ layer.register_parameter("input_scale", input_scale)
127
+
128
+ def apply_weights(self,
129
+ layer: torch.nn.Module,
130
+ x: torch.Tensor,
131
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
132
+
133
+ return apply_fp8_linear(
134
+ input=x,
135
+ weight=layer.weight,
136
+ weight_scale=layer.weight_scale,
137
+ input_scale=layer.input_scale,
138
+ bias=bias,
139
+ cutlass_fp8_supported=self.cutlass_fp8_supported,
140
+ use_per_token_if_dynamic=True)
vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py
new
+105
@@ -0,0 +1,105 @@
1
+from typing import Callable, List, Optional, Set
2
+
3
+import torch
4
+
5
+from vllm.logger import init_logger
6
+from vllm.model_executor.layers.quantization.kernels.scaled_mm import (
7
+ ScaledMMLinearLayerConfig, choose_scaled_mm_linear_kernel)
8
+from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
9
+from vllm.model_executor.parameter import (BasevLLMParameter,
10
+ ChannelQuantScaleParameter,
11
+ ModelWeightParameter,
12
+ PerTensorScaleParameter)
13
+
14
+logger = init_logger(__name__)
15
+
16
+
17
+class QuarkW8A8Int8(QuarkScheme):
18
+ _kernel_backends_being_used: Set[str] = set()
19
+
20
+ def __init__(self, qscheme: str, is_static_input_scheme: Optional[bool],
21
+ input_symmetric: Optional[bool]):
22
+ self.qscheme = qscheme
23
+ self.is_static_input_scheme = is_static_input_scheme
24
+ self.input_symmetric = input_symmetric
25
+
26
+ @classmethod
27
+ def get_min_capability(cls) -> int:
28
+ # turing and up
29
+ return 75
30
+
31
+ def create_weights(self, layer: torch.nn.Module,
32
+ output_partition_sizes: List[int],
33
+ input_size_per_partition: int,
34
+ params_dtype: torch.dtype, weight_loader: Callable,
35
+ **kwargs):
36
+ self.logical_widths = output_partition_sizes
37
+
38
+ scaled_mm_linear_kernel_config = ScaledMMLinearLayerConfig(
39
+ is_channelwise=(self.qscheme == "per_channel"),
40
+ is_static_input_scheme=(self.is_static_input_scheme is True),
41
+ input_symmetric=(self.input_symmetric is True))
42
+
43
+ kernel_type = choose_scaled_mm_linear_kernel(
44
+ scaled_mm_linear_kernel_config)
45
+
46
+ if kernel_type.__name__ not in self._kernel_backends_being_used:
47
+ logger.info("Using %s for QuarkW8A8Int8", kernel_type.__name__)
48
+ self._kernel_backends_being_used.add(kernel_type.__name__)
49
+
50
+ # WEIGHT
51
+ weight = ModelWeightParameter(data=torch.empty(
52
+ sum(output_partition_sizes),
53
+ input_size_per_partition,
54
+ dtype=torch.int8),
55
+ input_dim=1,
56
+ output_dim=0,
57
+ weight_loader=weight_loader)
58
+
59
+ layer.register_parameter("weight", weight)
60
+
61
+ # WEIGHT SCALE
62
+ if self.qscheme == "per_channel":
63
+ weight_scale = ChannelQuantScaleParameter(
64
+ data=torch.empty((sum(output_partition_sizes), 1),
65
+ dtype=torch.float32),
66
+ output_dim=0,
67
+ weight_loader=weight_loader)
68
+ else:
69
+ assert self.qscheme == "per_tensor"
70
+ weight_scale = PerTensorScaleParameter(data=torch.empty(
71
+ len(output_partition_sizes), dtype=torch.float32),
72
+ weight_loader=weight_loader)
73
+ layer.register_parameter("weight_scale", weight_scale)
74
+
75
+ # INPUT SCALE
76
+ if self.is_static_input_scheme:
77
+ input_scale = BasevLLMParameter(data=torch.empty(
78
+ 1, dtype=torch.float32),
79
+ weight_loader=weight_loader)
80
+ layer.register_parameter("input_scale", input_scale)
81
+
82
+ if not self.input_symmetric:
83
+ # Note: quark stores the zp using the same dtype
84
+ # as the weights
85
+ # AZP loaded as int8 but used as int32
86
+ input_zero_point = BasevLLMParameter(
87
+ data=torch.empty(1, dtype=torch.int8),
88
+ weight_loader=weight_loader)
89
+ layer.register_parameter("input_zero_point", input_zero_point)
90
+
91
+ self.kernel = kernel_type(c=scaled_mm_linear_kernel_config,
92
+ w_q_param_name="weight",
93
+ w_s_param_name="weight_scale",
94
+ i_s_param_name="input_scale",
95
+ i_zp_param_name="input_zero_point",
96
+ azp_adj_param_name="azp_adj")
97
+
98
+ # Checkpoints are serialized in quark format, which is
99
+ # different from the format the kernel may want. Handle repacking here.
100
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
101
+ self.kernel.process_weights_after_loading(layer)
102
+
103
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
104
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
105
+ return self.kernel.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/quark/utils.py
new
+99
@@ -0,0 +1,99 @@
1
+import re
2
+from typing import Any, Iterable, Optional
3
+
4
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
5
+ FUSED_LAYER_NAME_MAPPING)
6
+
7
+
8
+def deep_compare(dict1: Any, dict2: Any) -> bool:
9
+ if type(dict1) is not type(dict2):
10
+ return False
11
+ if isinstance(dict1, dict):
12
+ if dict1.keys() != dict2.keys():
13
+ return False
14
+ return all(deep_compare(dict1[k], dict2[k]) for k in dict1)
15
+ elif isinstance(dict1, list):
16
+ return set(dict1) == set(dict2)
17
+ else:
18
+ return dict1 == dict2
19
+
20
+
21
+def should_ignore_layer(layer_name: Optional[str],
22
+ ignore: Iterable[str]) -> bool:
23
+ if layer_name is None:
24
+ return False
25
+
26
+ # layer_name = model.layers.0.self_attn.qkv_proj
27
+ # proj_name = qkv_proj
28
+ proj_name = layer_name.split(".")[-1]
29
+
30
+ # Fused layers like gate_up_proj or qkv_proj will not be fused
31
+ # in the safetensors checkpoint. So, we convert the name
32
+ # from the fused version to unfused + check to make sure that
33
+ # each shard of the fused layer has the same scheme.
34
+ if proj_name in FUSED_LAYER_NAME_MAPPING:
35
+ shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
36
+
37
+ # Convert fused_name --> [shard_names]
38
+ shard_names = [
39
+ layer_name.replace(proj_name, shard_proj_name)
40
+ for shard_proj_name in shard_proj_names
41
+ ]
42
+
43
+ # Layer should be ignored if shards are ignored.
44
+ should_ignore_layer = None
45
+ for shard_name in shard_names:
46
+ should_ignore_shard = check_equal_or_regex_match(
47
+ layer_name=shard_name, targets=ignore)
48
+
49
+ # If shard_idx=0, set layer ignore to match shard.
50
+ if should_ignore_layer is None:
51
+ should_ignore_layer = should_ignore_shard
52
+
53
+ # If shard_idx=1+ confirm scheme matches prior shards.
54
+ elif should_ignore_shard != should_ignore_layer:
55
+ raise ValueError(f"Found a different quantization schemes for "
56
+ f"{shard_proj_names} in {layer_name}. vLLM "
57
+ "requires all to use the same scheme.")
58
+
59
+ # Unfused layers like down_proj and o_proj will match
60
+ # the safetensors checkpoint already.
61
+ else:
62
+ should_ignore_layer = check_equal_or_regex_match(layer_name=layer_name,
63
+ targets=ignore)
64
+
65
+ assert should_ignore_layer is not None
66
+ return should_ignore_layer
67
+
68
+
69
+def check_equal_or_regex_match(layer_name: str,
70
+ targets: Iterable[str]) -> bool:
71
+ """
72
+ Checks whether a layer_name is exactly equal or a regex match for
73
+ if target starts with 're:' to any target in list.
74
+ """
75
+ for target in targets:
76
+ if _is_equal_or_regex_match(layer_name, target):
77
+ return True
78
+ return False
79
+
80
+
81
+def _is_equal_or_regex_match(value: str,
82
+ target: str,
83
+ check_contains: bool = False) -> bool:
84
+ """
85
+ Checks whether a value is exactly equal or a regex match for target
86
+ if target starts with 're:'. If check_contains is set to True,
87
+ additionally checks if the target string is contained within the value.
88
+ """
89
+
90
+ if target.startswith("re:"):
91
+ pattern = target[3:]
92
+ if re.match(pattern, value):
93
+ return True
94
+ elif check_contains:
95
+ if target.lower() in value.lower():
96
+ return True
97
+ elif target == value:
98
+ return True
99
+ return False
vllm/model_executor/models/aria.py
+6
-5
@@ -13,8 +13,6 @@ from vllm.model_executor.layers.fused_moe import FusedMoE
13
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14
RowParallelLinear)
15
from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
17
- get_compressed_tensors_cache_scale)
16
from vllm.model_executor.layers.sampler import (SamplerOutput,
17
SamplingMetadata, get_sampler)
18
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
@@ -390,12 +388,15 @@ class AriaMoELMModel(LlamaModel):
388
# Models trained using ColossalAI may include these tensors in
389
# the checkpoint. Skip them.
390
continue
393
- if scale_name := get_compressed_tensors_cache_scale(name):
394
- # Loading kv cache scales for compressed-tensors quantization
391
+ if (self.quant_config is not None and
392
+ (scale_name := self.quant_config.get_cache_scale(name))):
393
+ # Loading kv cache scales for quark and
394
+ # compressed-tensors quantization
395
param = params_dict[scale_name]
396
weight_loader = getattr(param, "weight_loader",
397
default_weight_loader)
398
- loaded_weight = loaded_weight[0]
398
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
399
+ loaded_weight[0])
400
weight_loader(param, loaded_weight)
401
loaded_params.add(scale_name)
402
continue
vllm/model_executor/models/commandr.py
+14
@@ -437,6 +437,20 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
437
params_dict = dict(self.named_parameters())
438
loaded_params: Set[str] = set()
439
for name, loaded_weight in weights:
440
+
441
+ if (self.quant_config is not None and
442
+ (scale_name := self.quant_config.get_cache_scale(name))):
443
+ # Loading kv cache scales for quark and
444
+ # compressed-tensors quantization
445
+ param = params_dict[scale_name]
446
+ weight_loader = getattr(param, "weight_loader",
447
+ default_weight_loader)
448
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
449
+ loaded_weight[0])
450
+ weight_loader(param, loaded_weight)
451
+ loaded_params.add(scale_name)
452
+ continue
453
+
454
for param_name, shard_name, shard_id in stacked_params_mapping:
455
if shard_name not in name:
456
continue
vllm/model_executor/models/dbrx.py
+52
-21
@@ -83,7 +83,7 @@ class DbrxExperts(FusedMoE):
83
84
# Define custom weight loader for dbrx model
85
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
86
- weight_name: str):
86
+ weight_name: str, param_name: str):
87
tp_rank = get_tensor_model_parallel_rank()
88
param_data = param.data
89
shard_size = self.intermediate_size
@@ -91,25 +91,37 @@ class DbrxExperts(FusedMoE):
91
# DBRX uses GLU for each experts.
92
# GLU has 3 linear layers: w1, v1 and w2.
93
if weight_name.endswith("w1"):
94
- loaded_weight = torch.reshape(
95
- loaded_weight,
96
- [-1, self.intermediate_size * self.tp_size, self.d_model],
97
- )
98
- param_data[:, 0:shard_size, :] = loaded_weight[:, shard, :]
94
+ if param_name.endswith("weight"):
95
+ loaded_weight = torch.reshape(
96
+ loaded_weight,
97
+ [-1, self.intermediate_size * self.tp_size, self.d_model],
98
+ )
99
+ param_data[:, 0:shard_size, :] = loaded_weight[:, shard, :]
100
+ elif param_name.endswith("weight_scale"):
101
+ param_data[:, 0] = loaded_weight
102
+ else:
103
+ param_data = loaded_weight
104
if weight_name.endswith("v1"):
100
- loaded_weight = torch.reshape(
101
- loaded_weight,
102
- [-1, self.intermediate_size * self.tp_size, self.d_model],
103
- )
104
- param_data[:,
105
- shard_size:2 * shard_size, :] = loaded_weight[:,
106
- shard, :]
105
+ if param_name.endswith("weight"):
106
+ loaded_weight = torch.reshape(
107
+ loaded_weight,
108
+ [-1, self.intermediate_size * self.tp_size, self.d_model],
109
+ )
110
+ param_data[:, shard_size:2 *
111
+ shard_size, :] = loaded_weight[:, shard, :]
112
+ elif param_name.endswith("weight_scale"):
113
+ param_data[:, 1] = loaded_weight
114
+ else:
115
+ param_data[:] = loaded_weight
116
if weight_name.endswith("w2"):
108
- loaded_weight = torch.reshape(
109
- loaded_weight,
110
- [-1, self.intermediate_size * self.tp_size, self.d_model],
111
- ).transpose(1, 2)
112
- param_data[:] = loaded_weight[:, :, shard]
117
+ if param_name.endswith("weight"):
118
+ loaded_weight = torch.reshape(
119
+ loaded_weight,
120
+ [-1, self.intermediate_size * self.tp_size, self.d_model],
121
+ ).transpose(1, 2)
122
+ param_data[:] = loaded_weight[:, :, shard]
123
+ else:
124
+ param_data[:] = loaded_weight
125
126
127
class DbrxMoE(nn.Module):
@@ -430,14 +442,29 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
442
443
def load_weights(self, weights: Iterable[Tuple[str,
444
torch.Tensor]]) -> Set[str]:
433
-
445
expert_params_mapping = [(
435
- "w13_weight" if weight_name in ["w1", "v1"] else "w2_weight",
446
+ "w13" if weight_name in ["w1", "v1"] else "w2",
447
f"mlp.{weight_name}",
448
) for weight_name in ["w1", "v1", "w2"]]
449
params_dict = dict(self.named_parameters(remove_duplicate=False))
450
loaded_params: Set[str] = set()
451
+
452
for name, loaded_weight in weights:
453
+ if (self.quant_config is not None and
454
+ (scale_name := self.quant_config.get_cache_scale(name))):
455
+ # Loading kv cache scales for quark and
456
+ # compressed-tensors quantization
457
+ param = params_dict[scale_name]
458
+ weight_loader = getattr(param, "weight_loader",
459
+ default_weight_loader)
460
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
461
+ loaded_weight[0])
462
+ weight_loader(param, loaded_weight)
463
+ loaded_params.add(scale_name)
464
+ continue
465
+
466
+ if name.endswith(("w1", "w2", "v1")):
467
+ name = name + "_weight"
468
for param_name, weight_name in expert_params_mapping:
469
if weight_name not in name:
470
continue
@@ -446,8 +473,9 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
473
continue
474
param = params_dict[name]
475
weight_loader = param.weight_loader
449
- weight_loader(param, loaded_weight, weight_name)
476
+ weight_loader(param, loaded_weight, weight_name, name)
477
break
478
+
479
else:
480
# Remapping the name of FP8 kv-scale.
481
name = maybe_remap_kv_scale_name(name, params_dict)
@@ -456,6 +484,9 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
484
485
if is_pp_missing_parameter(name, self):
486
continue
487
+ name = maybe_remap_kv_scale_name(name, params_dict)
488
+ if name is None:
489
+ continue
490
param = params_dict[name]
491
weight_loader = getattr(param, "weight_loader",
492
default_weight_loader)
vllm/model_executor/models/exaone.py
+7
-5
@@ -39,8 +39,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39
RowParallelLinear)
40
from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
from vllm.model_executor.layers.quantization import QuantizationConfig
42
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43
- get_compressed_tensors_cache_scale)
42
from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -439,6 +437,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
437
438
self.config = config
439
self.lora_config = lora_config
440
+ self.quant_config = quant_config
441
442
self.transformer = ExaoneModel(
443
vllm_config=vllm_config,
@@ -532,12 +531,15 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
531
# processed with quantization, LoRA, fine-tuning, etc.
532
if self.config.tie_word_embeddings and "lm_head.weight" in name:
533
continue
535
- if scale_name := get_compressed_tensors_cache_scale(name):
536
- # Loading kv cache scales for compressed-tensors quantization
534
+ if (self.quant_config is not None and
535
+ (scale_name := self.quant_config.get_cache_scale(name))):
536
+ # Loading kv cache scales for quark and
537
+ # compressed-tensors quantization
538
param = params_dict[scale_name]
539
weight_loader = getattr(param, "weight_loader",
540
default_weight_loader)
540
- loaded_weight = loaded_weight[0]
541
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
542
+ loaded_weight[0])
543
weight_loader(param, loaded_weight)
544
loaded_params.add(scale_name)
545
continue
vllm/model_executor/models/gemma2.py
+3
-3
@@ -31,8 +31,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31
RowParallelLinear)
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
35
- get_compressed_tensors_cache_scale)
34
from vllm.model_executor.layers.rotary_embedding import get_rope
35
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -254,6 +252,7 @@ class Gemma2Model(nn.Module):
252
cache_config = vllm_config.cache_config
253
quant_config = vllm_config.quant_config
254
self.config = config
255
+ self.quant_config = quant_config
256
257
self.embed_tokens = VocabParallelEmbedding(
258
config.vocab_size,
@@ -329,7 +328,8 @@ class Gemma2Model(nn.Module):
328
params_dict = dict(self.named_parameters())
329
loaded_params: Set[str] = set()
330
for name, loaded_weight in weights:
332
- if scale_name := get_compressed_tensors_cache_scale(name):
331
+ if (self.quant_config is not None and
332
+ (scale_name := self.quant_config.get_cache_scale(name))):
333
# Loading kv cache scales for compressed-tensors quantization
334
param = params_dict[scale_name]
335
weight_loader = getattr(param, "weight_loader",
vllm/model_executor/models/gpt_j.py
+14
@@ -313,6 +313,20 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
313
for name, loaded_weight in weights:
314
if "attn.bias" in name or "attn.masked_bias" in name:
315
continue
316
+
317
+ if (self.quant_config is not None and
318
+ (scale_name := self.quant_config.get_cache_scale(name))):
319
+ # Loading kv cache scales for quark and
320
+ # compressed-tensors quantization
321
+ param = params_dict[scale_name]
322
+ weight_loader = getattr(param, "weight_loader",
323
+ default_weight_loader)
324
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
325
+ loaded_weight[0])
326
+ weight_loader(param, loaded_weight)
327
+ loaded_params.add(scale_name)
328
+ continue
329
+
330
for (param_name, weight_name, shard_id) in stacked_params_mapping:
331
if weight_name not in name:
332
continue
vllm/model_executor/models/granite.py
+7
-5
@@ -39,8 +39,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
from vllm.model_executor.layers.quantization.base_config import (
41
QuantizationConfig)
42
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43
- get_compressed_tensors_cache_scale)
42
from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -371,6 +369,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
369
370
self.config = config
371
self.lora_config = lora_config
372
+ self.quant_config = quant_config
373
374
self.model = GraniteModel(vllm_config=vllm_config,
375
prefix=maybe_prefix(prefix, "model"))
@@ -474,12 +473,15 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
473
# processed with quantization, LoRA, fine-tuning, etc.
474
if self.config.tie_word_embeddings and "lm_head.weight" in name:
475
continue
477
- if scale_name := get_compressed_tensors_cache_scale(name):
478
- # Loading kv cache scales for compressed-tensors quantization
476
+ if (self.quant_config is not None and
477
+ (scale_name := self.quant_config.get_cache_scale(name))):
478
+ # Loading kv cache scales for quark and
479
+ # compressed-tensors quantization
480
param = params_dict[scale_name]
481
weight_loader = getattr(param, "weight_loader",
482
default_weight_loader)
482
- loaded_weight = loaded_weight[0]
483
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
484
+ loaded_weight[0])
485
weight_loader(param, loaded_weight)
486
loaded_params.add(scale_name)
487
continue
vllm/model_executor/models/llama.py
+7
-5
@@ -38,8 +38,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
38
RowParallelLinear)
39
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
from vllm.model_executor.layers.quantization import QuantizationConfig
41
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
42
- get_compressed_tensors_cache_scale)
41
from vllm.model_executor.layers.rotary_embedding import get_rope
42
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -306,6 +304,7 @@ class LlamaModel(nn.Module):
304
lora_config = vllm_config.lora_config
305
306
self.config = config
307
+ self.quant_config = quant_config
308
self.padding_idx = config.pad_token_id
309
lora_vocab = (lora_config.lora_extra_vocab_size *
310
(lora_config.max_loras or 1)) if lora_config else 0
@@ -396,12 +395,15 @@ class LlamaModel(nn.Module):
395
# Models trained using ColossalAI may include these tensors in
396
# the checkpoint. Skip them.
397
continue
399
- if scale_name := get_compressed_tensors_cache_scale(name):
400
- # Loading kv cache scales for compressed-tensors quantization
398
+ if (self.quant_config is not None and
399
+ (scale_name := self.quant_config.get_cache_scale(name))):
400
+ # Loading kv cache scales for quark and
401
+ # compressed-tensors quantization
402
param = params_dict[scale_name]
403
weight_loader = getattr(param, "weight_loader",
404
default_weight_loader)
404
- loaded_weight = loaded_weight[0]
405
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
406
+ loaded_weight[0])
407
weight_loader(param, loaded_weight)
408
loaded_params.add(scale_name)
409
continue
vllm/model_executor/models/mixtral.py
+14
@@ -347,6 +347,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
347
lora_config = vllm_config.lora_config
348
self.config = config
349
self.lora_config = lora_config
350
+ self.quant_config = quant_config
351
352
self.model = MixtralModel(vllm_config=vllm_config,
353
prefix=maybe_prefix(prefix, "model"))
@@ -428,6 +429,19 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
429
if "rotary_emb.inv_freq" in name:
430
continue
431
432
+ if (self.quant_config is not None and
433
+ (scale_name := self.quant_config.get_cache_scale(name))):
434
+ # Loading kv cache scales for quark and
435
+ # compressed-tensors quantization
436
+ param = params_dict[scale_name]
437
+ weight_loader = getattr(param, "weight_loader",
438
+ default_weight_loader)
439
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
440
+ loaded_weight[0])
441
+ weight_loader(param, loaded_weight)
442
+ loaded_params.add(scale_name)
443
+ continue
444
+
445
for (param_name, weight_name, shard_id) in stacked_params_mapping:
446
if weight_name not in name:
447
continue
vllm/model_executor/models/mllama.py
+13
@@ -1116,6 +1116,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1116
super().__init__()
1117
config = vllm_config.model_config.hf_config
1118
quant_config = vllm_config.quant_config
1119
+ self.quant_config = quant_config
1120
self.vocab_size = config.text_config.vocab_size
1121
self.hidden_size = config.text_config.hidden_size
1122
self.max_num_tiles = config.vision_config.max_num_tiles
@@ -1429,6 +1430,18 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1430
name = name.replace('patch_embedding.weight',
1431
'patch_embedding._linear.weight')
1432
loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
1433
+ if (self.quant_config is not None and
1434
+ (scale_name := self.quant_config.get_cache_scale(name))):
1435
+ # Loading kv cache scales for quark and
1436
+ # compressed-tensors quantization
1437
+ param = params_dict[scale_name]
1438
+ weight_loader = getattr(param, "weight_loader",
1439
+ default_weight_loader)
1440
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
1441
+ loaded_weight[0])
1442
+ weight_loader(param, loaded_weight)
1443
+ updated_params.add(scale_name)
1444
+ continue
1445
for (param_name, weight_name, shard_id) in stacked_params_mapping:
1446
if weight_name not in name:
1447
continue
vllm/model_executor/models/nemotron.py
+13
@@ -405,6 +405,7 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
405
406
self.config = config
407
self.lora_config = lora_config
408
+ self.quant_config = quant_config
409
410
self.model = NemotronModel(vllm_config=vllm_config,
411
prefix=maybe_prefix(prefix, "model"))
@@ -489,6 +490,18 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
490
# Models trained using ColossalAI may include these tensors in
491
# the checkpoint. Skip them.
492
continue
493
+ if (self.quant_config is not None and
494
+ (scale_name := self.quant_config.get_cache_scale(name))):
495
+ # Loading kv cache scales for quark and
496
+ # compressed-tensors quantization
497
+ param = params_dict[scale_name]
498
+ weight_loader = getattr(param, "weight_loader",
499
+ default_weight_loader)
500
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
501
+ loaded_weight[0])
502
+ weight_loader(param, loaded_weight)
503
+ loaded_params.add(scale_name)
504
+ continue
505
for (param_name, weight_name, shard_id) in stacked_params_mapping:
506
if weight_name not in name:
507
continue
vllm/model_executor/models/phimoe.py
+14
@@ -546,6 +546,7 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
546
lora_config = vllm_config.lora_config
547
self.config = config
548
self.lora_config = lora_config
549
+ self.quant_config = vllm_config.quant_config
550
551
self.model = PhiMoEModel(vllm_config=vllm_config,
552
prefix=maybe_prefix(prefix, "model"))
@@ -623,6 +624,19 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
624
if "rotary_emb.inv_freq" in name:
625
continue
626
627
+ if (self.quant_config is not None and
628
+ (scale_name := self.quant_config.get_cache_scale(name))):
629
+ # Loading kv cache scales for quark and
630
+ # compressed-tensors quantization
631
+ param = params_dict[scale_name]
632
+ weight_loader = getattr(param, "weight_loader",
633
+ default_weight_loader)
634
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
635
+ loaded_weight[0])
636
+ weight_loader(param, loaded_weight)
637
+ loaded_params.add(scale_name)
638
+ continue
639
+
640
for param_name, weight_name, shard_id in stacked_params_mapping:
641
if weight_name not in name:
642
continue
vllm/model_executor/models/qwen2.py
+13
@@ -279,6 +279,7 @@ class Qwen2Model(nn.Module):
279
))
280
281
self.config = config
282
+ self.quant_config = quant_config
283
self.padding_idx = config.pad_token_id
284
self.vocab_size = config.vocab_size
285
@@ -364,6 +365,18 @@ class Qwen2Model(nn.Module):
365
for name, loaded_weight in weights:
366
if "rotary_emb.inv_freq" in name:
367
continue
368
+ if (self.quant_config is not None and
369
+ (scale_name := self.quant_config.get_cache_scale(name))):
370
+ # Loading kv cache scales for quark and
371
+ # compressed-tensors quantization
372
+ param = params_dict[scale_name]
373
+ weight_loader = getattr(param, "weight_loader",
374
+ default_weight_loader)
375
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
376
+ loaded_weight[0])
377
+ weight_loader(param, loaded_weight)
378
+ loaded_params.add(scale_name)
379
+ continue
380
for (param_name, weight_name, shard_id) in stacked_params_mapping:
381
if weight_name not in name:
382
continue
vllm/model_executor/models/solar.py
+7
-5
@@ -39,8 +39,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39
RowParallelLinear)
40
from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
from vllm.model_executor.layers.quantization import QuantizationConfig
42
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43
- get_compressed_tensors_cache_scale)
42
from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -409,6 +407,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
407
lora_config = vllm_config.lora_config
408
self.config = config
409
self.lora_config = lora_config
410
+ self.quant_config = quant_config
411
412
self.model = SolarModel(
413
vllm_config=vllm_config,
@@ -491,12 +490,15 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
490
# Models trained using ColossalAI may include these tensors in
491
# the checkpoint. Skip them.
492
continue
494
- if scale_name := get_compressed_tensors_cache_scale(name):
495
- # Loading kv cache scales for compressed-tensors quantization
493
+ if (self.quant_config is not None and
494
+ (scale_name := self.quant_config.get_cache_scale(name))):
495
+ # Loading kv cache scales for quark and
496
+ # compressed-tensors quantization
497
param = params_dict[scale_name]
498
weight_loader = getattr(param, "weight_loader",
499
default_weight_loader)
499
- loaded_weight = loaded_weight[0]
500
+ loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
501
+ loaded_weight[0])
502
weight_loader(param, loaded_weight)
503
loaded_params.add(scale_name)
504
continue
vllm/model_executor/parameter.py
+7
-1
@@ -56,8 +56,14 @@ class BasevLLMParameter(Parameter):
56
def weight_loader(self):
57
return self._weight_loader
58
59
+ def _is_1d_and_scalar(self, loaded_weight: torch.Tensor):
60
+ cond1 = self.data.ndim == 1 and self.data.numel() == 1
61
+ cond2 = loaded_weight.ndim == 0 and loaded_weight.numel() == 1
62
+ return (cond1 and cond2)
63
+
64
def _assert_and_load(self, loaded_weight: torch.Tensor):
60
- assert self.data.shape == loaded_weight.shape
65
+ assert (self.data.shape == loaded_weight.shape
66
+ or self._is_1d_and_scalar(loaded_weight))
67
self.data.copy_(loaded_weight)
68
69
def load_column_parallel_weight(self, loaded_weight: torch.Tensor):
vllm/platforms/rocm.py
+1
-1
@@ -70,7 +70,7 @@ class RocmPlatform(Platform):
70
71
supported_quantization: list[str] = [
72
"awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
73
- "fbgemm_fp8", "gguf"
73
+ "fbgemm_fp8", "gguf", "quark"
74
]
75
76
@classmethod