@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / de0526f668
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc][Quark] Upstream Quark format to VLLM (#10765)

Signed-off-by: kewang-xlnx <kewang@xilinx.com> Signed-off-by: kewang2 <kewang2@amd.com> Co-authored-by: kewang2 <kewang2@amd.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>

kewang-xlnx committed Jan 16, 2025 at 00:05 UTC de0526f668d6918c1884fd3b201308e9049e6be9
32 files changed +1264 -70
tests/quantization/test_quark.py new
+30
@@ -0,0 +1,30 @@
1 +"""Test model set-up and weight loading for quark-quantized models.
2 +
3 +Run `pytest tests/quantization/test_quark.py`.
4 +"""
5 +
6 +import torch
7 +
8 +from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
9 + QuarkLinearMethod, QuarkW8A8Fp8)
10 +
11 +
12 +def test_quark_fp8(vllm_runner):
13 + model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"
14 + with vllm_runner(model_path) as llm:
15 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
16 + layer = model.model.layers[0]
17 +
18 + qkv_proj = layer.self_attn.qkv_proj
19 +
20 + assert isinstance(qkv_proj.quant_method, QuarkLinearMethod)
21 + assert isinstance(qkv_proj.scheme, QuarkW8A8Fp8)
22 +
23 + if isinstance(qkv_proj.scheme, QuarkW8A8Fp8):
24 + assert len(qkv_proj.input_scale.shape) == 0
25 + assert qkv_proj.weight.dtype is torch.float8_e4m3fn
26 + #assert qkv_proj.weight.dtype is torch.float8_e4m3fnuz
27 + assert len(qkv_proj.weight_scale.shape) == 0
28 +
29 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
30 + assert output
vllm/config.py
+1 -1
@@ -553,7 +553,7 @@ class ModelConfig:
553 optimized_quantization_methods = [
554 "fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin",
555 "awq_marlin", "fbgemm_fp8", "compressed_tensors",
556 - "compressed-tensors", "experts_int8"
556 + "compressed-tensors", "experts_int8", "quark"
557 ]
558 if self.quantization is not None:
559 self.quantization = self.quantization.lower()
vllm/model_executor/layers/linear.py
+1 -1
@@ -32,7 +32,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
32 "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
33 "TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod",
34 "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod", "IPEXGPTQLinearMethod",
35 - "HQQMarlinMethod"
35 + "HQQMarlinMethod", "QuarkLinearMethod"
36 ]
37
38
vllm/model_executor/layers/quantization/__init__.py
+4
@@ -26,6 +26,7 @@ QUANTIZATION_METHODS: List[str] = [
26 "experts_int8",
27 "neuron_quant",
28 "ipex",
29 + "quark"
30 ]
31
32
@@ -34,6 +35,8 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
35 raise ValueError(f"Invalid quantization method: {quantization}")
36
37 # lazy import to avoid triggering `torch.compile` too early
38 + from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
39 +
40 from .aqlm import AQLMConfig
41 from .awq import AWQConfig
42 from .awq_marlin import AWQMarlinConfig
@@ -79,6 +82,7 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
82 "experts_int8": ExpertsInt8Config,
83 "neuron_quant": NeuronQuantConfig,
84 "ipex": IPEXConfig,
85 + "quark": QuarkConfig
86 }
87
88 return method_to_config[quantization]
vllm/model_executor/layers/quantization/base_config.py
+3
@@ -133,3 +133,6 @@ class QuantizationConfig(ABC):
133 method.
134 """
135 raise NotImplementedError
136 +
137 + def get_cache_scale(self, name: str) -> Optional[str]:
138 + return None
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+16
@@ -412,6 +412,22 @@ class CompressedTensorsConfig(QuantizationConfig):
412 self._check_scheme_supported(scheme.get_min_capability())
413 return scheme
414
415 + def get_cache_scale(self, name: str) -> Optional[str]:
416 + """
417 + Check whether the param name matches the format for k/v cache scales
418 + in compressed-tensors. If this is the case, return its equivalent
419 + param name expected by vLLM
420 +
421 + :param name: param name
422 + :return: matching param name for KV cache scale in vLLM
423 + """
424 + if name.endswith(".output_scale") and ".k_proj" in name:
425 + return name.replace(".k_proj.output_scale", ".attn.k_scale")
426 + if name.endswith(".output_scale") and ".v_proj" in name:
427 + return name.replace(".v_proj.output_scale", ".attn.v_scale")
428 + # If no matches, return None
429 + return None
430 +
431 @staticmethod
432 def supports_cutlass_24(
433 weight_quant: Optional[QuantizationArgs],
vllm/model_executor/layers/quantization/compressed_tensors/triton_scaled_mm.py
+4
@@ -136,6 +136,10 @@ def triton_scaled_mm(input: torch.Tensor,
136 assert N > 0 and K > 0 and M > 0
137 assert weight.shape[0] == K
138 assert input.dtype == weight.dtype
139 +
140 + scale_a = scale_a.reshape(-1, 1) if scale_a.dim() <= 1 else scale_a
141 + scale_b = scale_b.reshape(-1, 1) if scale_b.dim() <= 1 else scale_b
142 +
143 assert scale_a.dtype == scale_b.dtype and scale_a.is_floating_point()
144 assert scale_a.shape == torch.Size([1, 1]) or scale_a.shape == torch.Size(
145 [M, 1])
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
-17
@@ -133,23 +133,6 @@ def _find_first_match(value: str,
133 return None
134
135
136 -def get_compressed_tensors_cache_scale(name: str) -> Optional[str]:
137 - """
138 - Check whether the param name matches the format for k/v cache scales
139 - in compressed-tensors. If this is the case, return its equivalent
140 - param name expected by vLLM
141 -
142 - :param name: param name
143 - :return: matching param name for KV cache scale in vLLM
144 - """
145 - if name.endswith(".output_scale") and ".k_proj" in name:
146 - return name.replace(".k_proj.output_scale", ".attn.k_scale")
147 - if name.endswith(".output_scale") and ".v_proj" in name:
148 - return name.replace(".v_proj.output_scale", ".attn.v_scale")
149 - # If no matches, return None
150 - return None
151 -
152 -
136 def _is_equal_or_regex_match(value: str,
137 target: str,
138 check_contains: bool = False) -> bool:
vllm/model_executor/layers/quantization/quark/__init__.py
vllm/model_executor/layers/quantization/quark/quark.py new
+387
@@ -0,0 +1,387 @@
1 +import fnmatch
2 +import re
3 +from typing import Any, Dict, List, Optional, cast
4 +
5 +import torch
6 +
7 +from vllm.model_executor.layers.fused_moe import FusedMoE
8 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9 + UnquantizedLinearMethod)
10 +from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
11 + QuantizationConfig, QuantizeMethodBase)
12 +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
13 +from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
14 + QuarkMoEMethod)
15 +from vllm.model_executor.layers.quantization.quark.schemes import (
16 + QuarkScheme, QuarkW8A8Fp8, QuarkW8A8Int8)
17 +from vllm.model_executor.layers.quantization.quark.utils import (
18 + deep_compare, should_ignore_layer)
19 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
20 + FUSED_LAYER_NAME_MAPPING)
21 +from vllm.platforms import current_platform
22 +
23 +__all__ = ["QuarkLinearMethod"]
24 +
25 +
26 +class QuarkConfig(QuantizationConfig):
27 +
28 + def __init__(self,
29 + quant_config: Dict[str, Any],
30 + kv_cache_group: Optional[List[str]] = None,
31 + kv_cache_config: Optional[Dict[str, Any]] = None,
32 + pack_method: str = "reorder"):
33 + if kv_cache_group is None:
34 + kv_cache_group = []
35 + self.quant_config = quant_config
36 + self.kv_cache_group = kv_cache_group
37 + self.kv_cache_config = kv_cache_config
38 + self.pack_method = pack_method
39 +
40 + def get_linear_method(self) -> "QuarkLinearMethod":
41 + return QuarkLinearMethod(self)
42 +
43 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
44 + return [torch.float16, torch.bfloat16]
45 +
46 + @classmethod
47 + def get_min_capability(cls) -> int:
48 + return 70
49 +
50 + def get_name(self) -> str:
51 + return "quark"
52 +
53 + def get_quant_method(self, layer: torch.nn.Module,
54 + prefix: str) -> Optional["QuantizeMethodBase"]:
55 + from vllm.attention.layer import Attention # Avoid circular import
56 +
57 + # Check if the layer is skipped for quantization.
58 + exclude_layers = cast(List[str], self.quant_config.get("exclude"))
59 + if should_ignore_layer(prefix, ignore=exclude_layers):
60 + return UnquantizedLinearMethod()
61 + if isinstance(layer, LinearBase):
62 + scheme = self.get_scheme(layer=layer, layer_name=prefix)
63 + layer.scheme = scheme
64 + return QuarkLinearMethod(self)
65 + if isinstance(layer, Attention):
66 + return QuarkKVCacheMethod(self)
67 + if isinstance(layer, FusedMoE):
68 + return QuarkMoEMethod.get_moe_method(self,
69 + module=layer,
70 + layer_name=prefix)
71 + return None
72 +
73 + @classmethod
74 + def from_config(cls, config: Dict[str, Any]) -> "QuarkConfig":
75 + export_config = config.get("export")
76 + if export_config is None:
77 + raise ValueError("The export key should be included in "
78 + "the configurations of Quark quantized model")
79 + kv_cache_group = cast(List[str], export_config.get("kv_cache_group"))
80 + pack_method = cast(str, export_config.get("pack_method"))
81 +
82 + # In the export model of quark, the quantization configuration
83 + # of kv_cache is stored in layer_quant_config. First, it is
84 + # judged whether kv_cache_group exists, and then it is judged
85 + # whether layer_quant_config has a quantization configuration
86 + # that matches kv_cache.
87 + if len(kv_cache_group) == 0:
88 + kv_cache_config = None
89 + else:
90 + kv_cache_set = set(kv_cache_group)
91 + layer_quant_config = cast(Dict[str, Any],
92 + config.get("layer_quant_config"))
93 + layer_quant_names = list(layer_quant_config.keys())
94 + layer_quant_set = set(layer_quant_names)
95 +
96 + if not kv_cache_set.issubset(layer_quant_set):
97 + raise ValueError("The Quark quantized model has the "
98 + "kv_cache_group parameter setting, "
99 + "but no kv_cache quantization settings "
100 + "were found in the quantization "
101 + "configuration.")
102 +
103 + q_configs = [
104 + cast(Dict[str, Any], layer_quant_config.get(name))
105 + for name in kv_cache_group
106 + ]
107 + if not all(
108 + deep_compare(q_config, q_configs[0])
109 + for q_config in q_configs):
110 + raise ValueError(
111 + "The quantization method used for kv_cache should "
112 + "be the same, but the quantization method for the "
113 + "kv_cache layer in the config is different.")
114 + kv_cache_config = q_configs[0].get("output_tensors")
115 + if kv_cache_config is None:
116 + raise ValueError(
117 + "The kv_cache quantization configuration is empty.")
118 +
119 + # Since we have already set kv_cache quantization configurations,
120 + # we will remove the quantization configuration for the
121 + # output_tensors corresponding to the kv_cache layer.
122 + for q_config in q_configs:
123 + q_config["output_tensors"] = None
124 +
125 + return cls(quant_config=config,
126 + kv_cache_group=kv_cache_group,
127 + kv_cache_config=kv_cache_config,
128 + pack_method=pack_method)
129 +
130 + @classmethod
131 + def get_config_filenames(cls) -> List[str]:
132 + return []
133 +
134 + def _check_scheme_supported(self,
135 + min_capability: int,
136 + error: bool = True) -> bool:
137 + capability_tuple = current_platform.get_device_capability()
138 +
139 + if capability_tuple is not None:
140 + capability = capability_tuple.to_int()
141 + supported = capability >= min_capability
142 + if error and not supported:
143 + raise RuntimeError(
144 + "Quantization scheme is not supported for ",
145 + f"the current GPU. Min capability: {min_capability}. ",
146 + f"Current capability: {capability}.")
147 + return supported
148 + else:
149 + return False
150 +
151 + def _is_fp8_w8a8(self, weight_quant: Optional[Dict[str, Any]],
152 + input_quant: Optional[Dict[str, Any]]) -> bool:
153 + # Confirm weights and input quantized.
154 + if weight_quant is None or input_quant is None:
155 + return False
156 +
157 + # Confirm weight scheme is supported
158 + is_fp8_dtype = (weight_quant.get("dtype") == "fp8_e4m3"
159 + and input_quant.get("dtype") == "fp8_e4m3")
160 + is_static_weight = not weight_quant.get("is_dynamic")
161 + is_per_tensor_or_channel_weight = (weight_quant.get("qscheme")
162 + in ["per_tensor", "per_channel"])
163 +
164 + if not (is_fp8_dtype and is_static_weight
165 + and is_per_tensor_or_channel_weight):
166 + return False
167 +
168 + # Dynamic quantization is always supported if weights supported.
169 + if input_quant.get("is_dynamic"):
170 + return True
171 +
172 + # Confirm activation scheme is supported.
173 + is_per_tensor_activation = (input_quant.get("qscheme") == "per_tensor")
174 + return is_per_tensor_activation
175 +
176 + def _is_static_tensor_w8a8(self, weight_quant: Optional[Dict[str, Any]],
177 + input_quant: Optional[Dict[str, Any]]) -> bool:
178 + # Confirm weights and input quantized.
179 + if weight_quant is None or input_quant is None:
180 + return False
181 +
182 + is_int8_dtype = (weight_quant.get("dtype") == "int8"
183 + and input_quant.get("dtype") == "int8")
184 +
185 + is_tensor = (weight_quant.get("qscheme")
186 + in ["per_tensor", "per_channel"]
187 + and input_quant.get("qscheme") == "per_tensor")
188 +
189 + is_static = (not weight_quant.get("is_dynamic")
190 + and not input_quant.get("is_dynamic"))
191 +
192 + is_weight_symmetric = (weight_quant.get("symmetric") is True)
193 +
194 + # Both symmetric and asymmetric input quantization supported.
195 + # Only symmetric weight quantization supported.
196 + return is_int8_dtype and is_tensor and is_weight_symmetric and is_static
197 +
198 + def _find_matched_config(self, layer_name: str,
199 + module: torch.nn.Module) -> Dict[str, Any]:
200 +
201 + proj_name = layer_name.split(".")[-1]
202 + if proj_name in FUSED_LAYER_NAME_MAPPING:
203 + shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
204 +
205 + # Convert fused_name --> [shard_names]
206 + shard_names = [
207 + layer_name.replace(proj_name, shard_proj_name)
208 + for shard_proj_name in shard_proj_names
209 + ]
210 + shard_configs = [
211 + self._find_matched_config(shard_name, module)
212 + for shard_name in shard_names
213 + ]
214 + if not all(
215 + deep_compare(q_config, shard_configs[0])
216 + for q_config in shard_configs):
217 + raise ValueError(
218 + f"Found a different quantization configuration for "
219 + f"{shard_proj_names} in {layer_name}. vLLM "
220 + "requires all to use the same scheme.")
221 + return shard_configs[0]
222 + else:
223 + layer_quant_config = cast(
224 + Dict[str, Any], self.quant_config.get("layer_quant_config"))
225 + for name_pattern in layer_quant_config:
226 + if fnmatch.fnmatch(layer_name, name_pattern):
227 + return layer_quant_config[name_pattern]
228 +
229 + layer_type = cast(str, type(module))
230 + layer_type_quant_config = cast(
231 + Dict[str, Any],
232 + self.quant_config.get("layer_type_quant_config"))
233 + if layer_type in layer_type_quant_config:
234 + return layer_type_quant_config[layer_type]
235 +
236 + global_quant_config = cast(
237 + Dict[str, Any], self.quant_config.get("global_quant_config"))
238 + return global_quant_config
239 +
240 + def _get_scheme_from_config(self, config: Dict[str, Any]) -> "QuarkScheme":
241 + if config.get("output_tensors") or config.get("bias"):
242 + raise NotImplementedError(
243 + "Currently, Quark models with output_tensors "
244 + "and bias quantized are not supported")
245 + weight_config = cast(Dict[str, Any], config.get("weight"))
246 + input_config = cast(Dict[str, Any], config.get("input_tensors"))
247 +
248 + if self._is_fp8_w8a8(weight_config, input_config):
249 + is_fp8_w8a8_supported = self._check_scheme_supported(
250 + QuarkW8A8Fp8.get_min_capability(), error=False)
251 + if is_fp8_w8a8_supported:
252 + weight_qscheme = cast(str, weight_config.get("qscheme"))
253 + input_static = (input_config is not None and
254 + not cast(bool, input_config.get("is_dynamic")))
255 + return QuarkW8A8Fp8(qscheme=weight_qscheme,
256 + is_static_input_scheme=input_static)
257 + elif self._is_static_tensor_w8a8(weight_config, input_config):
258 + weight_qscheme = cast(str, weight_config.get("qscheme"))
259 + return QuarkW8A8Int8(qscheme=weight_qscheme,
260 + is_static_input_scheme=True,
261 + input_symmetric=input_config.get("symmetric"))
262 +
263 + raise NotImplementedError("No quark compatible scheme was found. "
264 + f"Weight config: {weight_config}, "
265 + f"Input config: {input_config}")
266 +
267 + def get_scheme(self, layer: torch.nn.Module,
268 + layer_name: str) -> "QuarkScheme":
269 +
270 + layer_quant_config = self._find_matched_config(layer_name, layer)
271 +
272 + # Find the quant_scheme
273 + scheme = self._get_scheme_from_config(layer_quant_config)
274 + # Raise error if device does not support the scheme
275 + # (e.g. fp8 needs ada lovelace)
276 + self._check_scheme_supported(scheme.get_min_capability())
277 +
278 + return scheme
279 +
280 + def get_cache_scale(self, name: str) -> Optional[str]:
281 + """
282 + Check whether the param name matches the format for k/v cache scales
283 + in quark. If this is the case, return its equivalent param name
284 + expected by vLLM
285 +
286 + :param name: param name
287 + :return: matching param name for KV cache scale in vLLM
288 + """
289 + if self.kv_cache_group is None or len(self.kv_cache_group) == 0:
290 + return None
291 +
292 + kv_proj_names = [
293 + re.split(r"[*.]", kv_cache)[-1] for kv_cache in self.kv_cache_group
294 + ]
295 + if name.endswith(".output_scale"):
296 + if len(kv_proj_names) == 1 and kv_proj_names[0] in name:
297 + kv_output_scale_name = "." + kv_proj_names[0] + ".output_scale"
298 + return name.replace(kv_output_scale_name, ".attn.k_scale")
299 +
300 + elif len(kv_proj_names) == 2:
301 + for kv_proj_name in kv_proj_names:
302 + if kv_proj_name in name and kv_proj_name == "k_proj":
303 + return name.replace(".k_proj.output_scale",
304 + ".attn.k_scale")
305 + elif kv_proj_name in name and kv_proj_name == "v_proj":
306 + return name.replace(".v_proj.output_scale",
307 + ".attn.v_scale")
308 +
309 + # If no matches, return None
310 + return None
311 +
312 +
313 +class QuarkLinearMethod(LinearMethodBase):
314 +
315 + def __init__(self, quantization_config: QuarkConfig):
316 + self.quantization_config = quantization_config
317 +
318 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
319 + layer.scheme.process_weights_after_loading(layer)
320 +
321 + def create_weights(self, layer: torch.nn.Module,
322 + input_size_per_partition: int,
323 + output_partition_sizes: List[int], input_size: int,
324 + output_size: int, params_dtype: torch.dtype,
325 + **extra_weight_attrs):
326 + """
327 + Use the CompressedTensorsScheme associated with each layer to create
328 + the necessary parameters for the layer. See LinearMethodBase for param
329 + details
330 + """
331 + weight_loader = extra_weight_attrs.get("weight_loader")
332 + layer.scheme.create_weights(
333 + layer=layer,
334 + input_size=input_size,
335 + input_size_per_partition=input_size_per_partition,
336 + output_partition_sizes=output_partition_sizes,
337 + output_size=output_size,
338 + params_dtype=params_dtype,
339 + weight_loader=weight_loader)
340 +
341 + def apply(self,
342 + layer: torch.nn.Module,
343 + x: torch.Tensor,
344 + bias: Optional[torch.Tensor] = None):
345 + """
346 + Use the output of create_weights and the CompressedTensorsScheme
347 + associated with the layer to apply the forward pass with the
348 + layer input. See LinearMethodBase for param details
349 +
350 + """
351 + scheme = layer.scheme
352 + if scheme is None:
353 + raise ValueError("A scheme must be defined for each layer")
354 + return scheme.apply_weights(layer, x, bias=bias)
355 +
356 +
357 +class QuarkKVCacheMethod(BaseKVCacheMethod):
358 + """
359 + Supports loading kv-cache scaling factors from quark checkpoints.
360 + """
361 +
362 + def __init__(self, quant_config: QuarkConfig):
363 + self.validate_kv_cache_config(quant_config.kv_cache_config)
364 + super().__init__(quant_config)
365 +
366 + @staticmethod
367 + def validate_kv_cache_config(kv_cache_config: Optional[Dict[str, Any]]):
368 + """
369 + Validator for the kv cache configuration. Useful for controlling the
370 + kv cache quantization schemes, that are being supported in vLLM
371 + :param kv_cache_config: the quark kv cache scheme
372 + """
373 + if kv_cache_config is None:
374 + return
375 +
376 + dtype = kv_cache_config.get("dtype")
377 + if dtype != "fp8_e4m3":
378 + raise NotImplementedError(
379 + "Currently supported kv cache quantization is "
380 + f"dtype=fp8_e4m3, however received {dtype}")
381 +
382 + qscheme = kv_cache_config.get("qscheme")
383 + if qscheme != "per_tensor":
384 + raise NotImplementedError(
385 + "Only support per-tensor scaling factor "
386 + "for quark KV cache. "
387 + f"Expected qscheme: per_tensor, found qscheme: {qscheme}")
vllm/model_executor/layers/quantization/quark/quark_moe.py new
+225
@@ -0,0 +1,225 @@
1 +from typing import Any, Callable, Dict, Optional
2 +
3 +import torch
4 +
5 +import vllm.model_executor.layers.fused_moe # noqa
6 +from vllm import _custom_ops as ops
7 +from vllm.logger import init_logger
8 +from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
9 + FusedMoeWeightScaleSupported)
10 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11 + all_close_1d, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize)
12 +from vllm.model_executor.utils import set_weight_attrs
13 +from vllm.platforms import current_platform
14 +
15 +logger = init_logger(__name__)
16 +
17 +__all__ = ["QuarkMoEMethod", "QuarkW8A8Fp8MoEMethod"]
18 +
19 +
20 +class QuarkMoEMethod(FusedMoEMethodBase):
21 +
22 + @staticmethod
23 + def get_moe_method(
24 + quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821
25 + module: torch.nn.Module,
26 + layer_name: str) -> "QuarkMoEMethod":
27 + layer_quant_config = quant_config._find_matched_config(
28 + layer_name, module)
29 +
30 + if (layer_quant_config.get("output_tensors")
31 + or layer_quant_config.get("bias")):
32 + raise NotImplementedError("Currently, Quark models with "
33 + "output_tensors and bias "
34 + "quantized are not supported")
35 + weight_config = layer_quant_config.get("weight")
36 + input_config = layer_quant_config.get("input_tensors")
37 +
38 + if quant_config._is_fp8_w8a8(weight_config, input_config):
39 + return QuarkW8A8Fp8MoEMethod(weight_config, input_config)
40 + else:
41 + raise RuntimeError("Unsupported FusedMoe scheme")
42 +
43 +
44 +class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
45 +
46 + def __init__(self, weight_config: Dict[str, Any], input_config: Dict[str,
47 + Any]):
48 + self.weight_quant = weight_config
49 + self.input_quant = input_config
50 +
51 + weight_qscheme = self.weight_quant.get("qscheme")
52 + input_qscheme = self.input_quant.get("qscheme")
53 + if not (weight_qscheme == "per_tensor"
54 + and input_qscheme == "per_tensor"):
55 + raise ValueError(
56 + "For FP8 Fused MoE layers, only per-tensor scales"
57 + "for weights and activations are supported. Found "
58 + f"{weight_qscheme}, {input_qscheme}") # noqa E501
59 +
60 + self.static_input_scales = not self.input_quant.get("is_dynamic")
61 +
62 + def create_weights(self, layer: torch.nn.Module, num_experts: int,
63 + hidden_size: int, intermediate_size: int,
64 + params_dtype: torch.dtype, **extra_weight_attrs):
65 +
66 + params_dtype = torch.float8_e4m3fn
67 +
68 + # WEIGHTS
69 + w13_weight = torch.nn.Parameter(torch.empty(num_experts,
70 + 2 * intermediate_size,
71 + hidden_size,
72 + dtype=params_dtype),
73 + requires_grad=False)
74 + layer.register_parameter("w13_weight", w13_weight)
75 + set_weight_attrs(w13_weight, extra_weight_attrs)
76 +
77 + w2_weight = torch.nn.Parameter(torch.empty(num_experts,
78 + hidden_size,
79 + intermediate_size,
80 + dtype=params_dtype),
81 + requires_grad=False)
82 + layer.register_parameter("w2_weight", w2_weight)
83 + set_weight_attrs(w2_weight, extra_weight_attrs)
84 +
85 + # WEIGHT_SCALES
86 + # Allocate 2 scales for w1 and w3 respectively.
87 + # They will be combined to a single scale after weight loading.
88 + w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
89 + 2,
90 + dtype=torch.float32),
91 + requires_grad=False)
92 + layer.register_parameter("w13_weight_scale", w13_weight_scale)
93 +
94 + w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
95 + dtype=torch.float32),
96 + requires_grad=False)
97 + layer.register_parameter("w2_weight_scale", w2_weight_scale)
98 + # Add the quantization method used (per tensor/grouped/channel)
99 + # to ensure the weight scales are loaded in properly
100 + extra_weight_attrs.update(
101 + {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value})
102 + set_weight_attrs(w13_weight_scale, extra_weight_attrs)
103 + set_weight_attrs(w2_weight_scale, extra_weight_attrs)
104 +
105 + # INPUT_SCALES
106 + if self.static_input_scales:
107 + w13_input_scale = torch.nn.Parameter(torch.ones(
108 + num_experts, dtype=torch.float32),
109 + requires_grad=False)
110 + layer.register_parameter("w13_input_scale", w13_input_scale)
111 + set_weight_attrs(w13_input_scale, extra_weight_attrs)
112 +
113 + w2_input_scale = torch.nn.Parameter(torch.ones(
114 + num_experts, dtype=torch.float32),
115 + requires_grad=False)
116 + layer.register_parameter("w2_input_scale", w2_input_scale)
117 + set_weight_attrs(w2_input_scale, extra_weight_attrs)
118 + else:
119 + layer.w13_input_scale = None
120 + layer.w2_input_scale = None
121 +
122 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
123 + # Fp8 moe kernels require a single activation scale.
124 + # We take the max of all the scales in case they differ.
125 + if self.static_input_scales:
126 + if (layer.w13_input_scale is None or layer.w2_input_scale is None):
127 + raise ValueError(
128 + "QuantConfig has static quantization, but found "
129 + "activation scales are None.")
130 + if (not all_close_1d(layer.w13_input_scale)
131 + or not all_close_1d(layer.w2_input_scale)):
132 + logger.warning_once(
133 + "Found input_scales that are not equal for "
134 + "fp8 MoE layer. Using the maximum across experts "
135 + "for each layer. ")
136 + layer.w13_input_scale = torch.nn.Parameter(
137 + layer.w13_input_scale.max(), requires_grad=False)
138 + layer.w2_input_scale = torch.nn.Parameter(
139 + layer.w2_input_scale.max(), requires_grad=False)
140 +
141 + # If rocm, normalize the weights and scales to e4m3fnuz
142 + if current_platform.is_rocm():
143 + # Normalize the weights and scales
144 + w13_weight, w13_weight_scale, w13_input_scale = \
145 + normalize_e4m3fn_to_e4m3fnuz(
146 + layer.w13_weight, layer.w13_weight_scale,
147 + layer.w13_input_scale)
148 + w2_weight, w2_weight_scale, w2_input_scale = \
149 + normalize_e4m3fn_to_e4m3fnuz(
150 + layer.w2_weight, layer.w2_weight_scale,
151 + layer.w2_input_scale)
152 + # Reset the parameter
153 + layer.w13_weight = torch.nn.Parameter(w13_weight,
154 + requires_grad=False)
155 + layer.w13_weight_scale = torch.nn.Parameter(w13_weight_scale,
156 + requires_grad=False)
157 + if w13_input_scale is not None:
158 + layer.w13_input_scale = torch.nn.Parameter(w13_input_scale,
159 + requires_grad=False)
160 + layer.w2_weight = torch.nn.Parameter(w2_weight,
161 + requires_grad=False)
162 + layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale,
163 + requires_grad=False)
164 + if w2_input_scale is not None:
165 + layer.w2_input_scale = torch.nn.Parameter(w2_input_scale,
166 + requires_grad=False)
167 +
168 + # Fp8 moe kernel needs single weight scale for w13 per expert.
169 + # We take the max then dequant and requant each expert.
170 + assert layer.w13_weight_scale is not None
171 + shard_size = layer.intermediate_size_per_partition
172 + max_w13_scales = layer.w13_weight_scale.max(dim=1).values
173 + for expert_id in range(layer.num_experts):
174 + start = 0
175 + for shard_id in range(2):
176 + dq_weight = per_tensor_dequantize(
177 + layer.w13_weight[expert_id][start:start + shard_size, :],
178 + layer.w13_weight_scale[expert_id][shard_id])
179 + layer.w13_weight[expert_id][
180 + start:start + shard_size, :], _ = ops.scaled_fp8_quant(
181 + dq_weight, max_w13_scales[expert_id])
182 + start += shard_size
183 +
184 + layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales,
185 + requires_grad=False)
186 +
187 + def apply(
188 + self,
189 + layer: torch.nn.Module,
190 + x: torch.Tensor,
191 + router_logits: torch.Tensor,
192 + top_k: int,
193 + renormalize: bool,
194 + use_grouped_topk: bool = False,
195 + topk_group: Optional[int] = None,
196 + num_expert_group: Optional[int] = None,
197 + custom_routing_function: Optional[Callable] = None,
198 + scoring_func: str = "softmax",
199 + e_score_correction_bias: Optional[torch.Tensor] = None,
200 + ) -> torch.Tensor:
201 + from vllm.model_executor.layers.fused_moe import fused_experts
202 +
203 + topk_weights, topk_ids = FusedMoE.select_experts(
204 + hidden_states=x,
205 + router_logits=router_logits,
206 + use_grouped_topk=use_grouped_topk,
207 + top_k=top_k,
208 + renormalize=renormalize,
209 + topk_group=topk_group,
210 + num_expert_group=num_expert_group,
211 + custom_routing_function=custom_routing_function,
212 + scoring_func=scoring_func,
213 + e_score_correction_bias=e_score_correction_bias)
214 +
215 + return fused_experts(x,
216 + layer.w13_weight,
217 + layer.w2_weight,
218 + topk_weights=topk_weights,
219 + topk_ids=topk_ids,
220 + inplace=True,
221 + use_fp8_w8a8=True,
222 + w1_scale=layer.w13_weight_scale,
223 + w2_scale=layer.w2_weight_scale,
224 + a1_scale=layer.w13_input_scale,
225 + a2_scale=layer.w2_input_scale)
vllm/model_executor/layers/quantization/quark/schemes/__init__.py new
+5
@@ -0,0 +1,5 @@
1 +from .quark_scheme import QuarkScheme
2 +from .quark_w8a8_fp8 import QuarkW8A8Fp8
3 +from .quark_w8a8_int8 import QuarkW8A8Int8
4 +
5 +__all__ = ["QuarkScheme", "QuarkW8A8Fp8", "QuarkW8A8Int8"]
vllm/model_executor/layers/quantization/quark/schemes/quark_scheme.py new
+52
@@ -0,0 +1,52 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Optional
3 +
4 +import torch
5 +
6 +__all__ = ["QuarkScheme"]
7 +
8 +
9 +class QuarkScheme(ABC):
10 + """
11 + Abstract class used to describe the weight creation and forward pass
12 + of different quantization schemes supported by Quark.
13 + """
14 +
15 + @classmethod
16 + @abstractmethod
17 + def get_min_capability(cls) -> int:
18 + """
19 + Get minimum device capability.
20 + """
21 + raise NotImplementedError
22 +
23 + @abstractmethod
24 + def create_weights(self, *args, **kwargs):
25 + """
26 + Weight creation for the particular scheme. Inputs to this function
27 +
28 + """
29 + raise NotImplementedError
30 +
31 + @abstractmethod
32 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
33 + bias: Optional[torch.Tensor]):
34 + """
35 + Run the forward pass for the particular scheme. This is where
36 + scheme-specific dequant/quant steps/kernels should be applied.
37 +
38 + :param layer: torch.nn.Module with the registered weights and
39 + other parameters relevant to the particular scheme.
40 + :param x: input to the layer
41 + :param bias: bias parameter
42 +
43 + """
44 + raise NotImplementedError
45 +
46 + @abstractmethod
47 + def process_weights_after_loading(self, layer: torch.nn.Module):
48 + """
49 + Called after weight loading is complete for any cleanup that
50 + needs to occur.
51 + """
52 + raise NotImplementedError
vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py new
+140
@@ -0,0 +1,140 @@
1 +from typing import Callable, List, Optional
2 +
3 +import torch
4 +from torch.nn import Parameter
5 +
6 +from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
7 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
8 + apply_fp8_linear, cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz,
9 + requantize_with_max_scale)
10 +from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
11 + ModelWeightParameter,
12 + PerTensorScaleParameter)
13 +from vllm.platforms import current_platform
14 +
15 +__all__ = ["QuarkW8A8Fp8"]
16 +
17 +
18 +class QuarkW8A8Fp8(QuarkScheme):
19 +
20 + def __init__(self, qscheme: str, is_static_input_scheme: Optional[bool]):
21 + self.qscheme = qscheme
22 + self.is_static_input_scheme = is_static_input_scheme
23 + self.cutlass_fp8_supported = cutlass_fp8_supported()
24 +
25 + @classmethod
26 + def get_min_capability(cls) -> int:
27 + # lovelace and up
28 + return 89
29 +
30 + def process_weights_after_loading(self, layer) -> None:
31 + # If per tensor, when we have a fused module (e.g. QKV) with per
32 + # tensor scales (thus N scales being passed to the kernel),
33 + # requantize so we can always run per tensor
34 + if self.qscheme == "per_tensor":
35 + max_w_scale, weight = requantize_with_max_scale(
36 + weight=layer.weight,
37 + weight_scale=layer.weight_scale,
38 + logical_widths=layer.logical_widths,
39 + )
40 +
41 + if current_platform.is_rocm():
42 + weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
43 + weight=weight,
44 + weight_scale=max_w_scale,
45 + input_scale=layer.input_scale)
46 + if input_scale is not None:
47 + layer.input_scale = Parameter(input_scale,
48 + requires_grad=False)
49 +
50 + layer.weight = Parameter(weight.t(), requires_grad=False)
51 + layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
52 +
53 + # If channelwise, scales are already lined up, so just transpose.
54 + elif self.qscheme == "per_channel":
55 + weight = layer.weight
56 +
57 + if current_platform.is_rocm():
58 + weight, weight_scale, input_scale = \
59 + normalize_e4m3fn_to_e4m3fnuz(
60 + weight=weight,
61 + weight_scale=layer.weight_scale,
62 + input_scale=layer.input_scale)
63 + if input_scale is not None:
64 + layer.input_scale = Parameter(input_scale,
65 + requires_grad=False)
66 + else:
67 + weight_scale = layer.weight_scale.data
68 +
69 + layer.weight = Parameter(weight.t(), requires_grad=False)
70 + # required by torch.compile to be torch.nn.Parameter
71 + layer.weight_scale = Parameter(weight_scale, requires_grad=False)
72 +
73 + else:
74 + raise ValueError(f"Unknown quantization scheme {self.qscheme}")
75 +
76 + # INPUT SCALE
77 + if self.is_static_input_scheme:
78 + layer.input_scale = Parameter(layer.input_scale.max(),
79 + requires_grad=False)
80 + else:
81 + layer.input_scale = None
82 +
83 + def create_weights(self, layer: torch.nn.Module,
84 + output_partition_sizes: List[int],
85 + input_size_per_partition: int,
86 + params_dtype: torch.dtype, weight_loader: Callable,
87 + **kwargs):
88 + output_size_per_partition = sum(output_partition_sizes)
89 + layer.logical_widths = output_partition_sizes
90 +
91 + # WEIGHT
92 + weight = ModelWeightParameter(data=torch.empty(
93 + output_size_per_partition,
94 + input_size_per_partition,
95 + dtype=torch.float8_e4m3fn),
96 + input_dim=1,
97 + output_dim=0,
98 + weight_loader=weight_loader)
99 + layer.register_parameter("weight", weight)
100 +
101 + # WEIGHT SCALE
102 + # TODO: update create_xxx_parameter functions to return
103 + # the newly added parameters
104 + if self.qscheme == "per_channel":
105 + weight_scale = ChannelQuantScaleParameter(
106 + data=torch.empty((sum(output_partition_sizes), 1),
107 + dtype=torch.float32),
108 + output_dim=0,
109 + weight_loader=weight_loader)
110 + else:
111 + assert self.qscheme == "per_tensor"
112 + weight_scale = PerTensorScaleParameter(data=torch.empty(
113 + len(output_partition_sizes), dtype=torch.float32),
114 + weight_loader=weight_loader)
115 +
116 + # min requirement for fp8 kernels
117 + weight_scale[:] = torch.finfo(torch.float32).min
118 + layer.register_parameter("weight_scale", weight_scale)
119 +
120 + # INPUT SCALE
121 + if self.is_static_input_scheme:
122 + input_scale = PerTensorScaleParameter(data=torch.empty(
123 + len(output_partition_sizes), dtype=torch.float32),
124 + weight_loader=weight_loader)
125 + input_scale[:] = torch.finfo(torch.float32).min
126 + layer.register_parameter("input_scale", input_scale)
127 +
128 + def apply_weights(self,
129 + layer: torch.nn.Module,
130 + x: torch.Tensor,
131 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
132 +
133 + return apply_fp8_linear(
134 + input=x,
135 + weight=layer.weight,
136 + weight_scale=layer.weight_scale,
137 + input_scale=layer.input_scale,
138 + bias=bias,
139 + cutlass_fp8_supported=self.cutlass_fp8_supported,
140 + use_per_token_if_dynamic=True)
vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py new
+105
@@ -0,0 +1,105 @@
1 +from typing import Callable, List, Optional, Set
2 +
3 +import torch
4 +
5 +from vllm.logger import init_logger
6 +from vllm.model_executor.layers.quantization.kernels.scaled_mm import (
7 + ScaledMMLinearLayerConfig, choose_scaled_mm_linear_kernel)
8 +from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
9 +from vllm.model_executor.parameter import (BasevLLMParameter,
10 + ChannelQuantScaleParameter,
11 + ModelWeightParameter,
12 + PerTensorScaleParameter)
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +class QuarkW8A8Int8(QuarkScheme):
18 + _kernel_backends_being_used: Set[str] = set()
19 +
20 + def __init__(self, qscheme: str, is_static_input_scheme: Optional[bool],
21 + input_symmetric: Optional[bool]):
22 + self.qscheme = qscheme
23 + self.is_static_input_scheme = is_static_input_scheme
24 + self.input_symmetric = input_symmetric
25 +
26 + @classmethod
27 + def get_min_capability(cls) -> int:
28 + # turing and up
29 + return 75
30 +
31 + def create_weights(self, layer: torch.nn.Module,
32 + output_partition_sizes: List[int],
33 + input_size_per_partition: int,
34 + params_dtype: torch.dtype, weight_loader: Callable,
35 + **kwargs):
36 + self.logical_widths = output_partition_sizes
37 +
38 + scaled_mm_linear_kernel_config = ScaledMMLinearLayerConfig(
39 + is_channelwise=(self.qscheme == "per_channel"),
40 + is_static_input_scheme=(self.is_static_input_scheme is True),
41 + input_symmetric=(self.input_symmetric is True))
42 +
43 + kernel_type = choose_scaled_mm_linear_kernel(
44 + scaled_mm_linear_kernel_config)
45 +
46 + if kernel_type.__name__ not in self._kernel_backends_being_used:
47 + logger.info("Using %s for QuarkW8A8Int8", kernel_type.__name__)
48 + self._kernel_backends_being_used.add(kernel_type.__name__)
49 +
50 + # WEIGHT
51 + weight = ModelWeightParameter(data=torch.empty(
52 + sum(output_partition_sizes),
53 + input_size_per_partition,
54 + dtype=torch.int8),
55 + input_dim=1,
56 + output_dim=0,
57 + weight_loader=weight_loader)
58 +
59 + layer.register_parameter("weight", weight)
60 +
61 + # WEIGHT SCALE
62 + if self.qscheme == "per_channel":
63 + weight_scale = ChannelQuantScaleParameter(
64 + data=torch.empty((sum(output_partition_sizes), 1),
65 + dtype=torch.float32),
66 + output_dim=0,
67 + weight_loader=weight_loader)
68 + else:
69 + assert self.qscheme == "per_tensor"
70 + weight_scale = PerTensorScaleParameter(data=torch.empty(
71 + len(output_partition_sizes), dtype=torch.float32),
72 + weight_loader=weight_loader)
73 + layer.register_parameter("weight_scale", weight_scale)
74 +
75 + # INPUT SCALE
76 + if self.is_static_input_scheme:
77 + input_scale = BasevLLMParameter(data=torch.empty(
78 + 1, dtype=torch.float32),
79 + weight_loader=weight_loader)
80 + layer.register_parameter("input_scale", input_scale)
81 +
82 + if not self.input_symmetric:
83 + # Note: quark stores the zp using the same dtype
84 + # as the weights
85 + # AZP loaded as int8 but used as int32
86 + input_zero_point = BasevLLMParameter(
87 + data=torch.empty(1, dtype=torch.int8),
88 + weight_loader=weight_loader)
89 + layer.register_parameter("input_zero_point", input_zero_point)
90 +
91 + self.kernel = kernel_type(c=scaled_mm_linear_kernel_config,
92 + w_q_param_name="weight",
93 + w_s_param_name="weight_scale",
94 + i_s_param_name="input_scale",
95 + i_zp_param_name="input_zero_point",
96 + azp_adj_param_name="azp_adj")
97 +
98 + # Checkpoints are serialized in quark format, which is
99 + # different from the format the kernel may want. Handle repacking here.
100 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
101 + self.kernel.process_weights_after_loading(layer)
102 +
103 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
104 + bias: Optional[torch.Tensor]) -> torch.Tensor:
105 + return self.kernel.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/quark/utils.py new
+99
@@ -0,0 +1,99 @@
1 +import re
2 +from typing import Any, Iterable, Optional
3 +
4 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
5 + FUSED_LAYER_NAME_MAPPING)
6 +
7 +
8 +def deep_compare(dict1: Any, dict2: Any) -> bool:
9 + if type(dict1) is not type(dict2):
10 + return False
11 + if isinstance(dict1, dict):
12 + if dict1.keys() != dict2.keys():
13 + return False
14 + return all(deep_compare(dict1[k], dict2[k]) for k in dict1)
15 + elif isinstance(dict1, list):
16 + return set(dict1) == set(dict2)
17 + else:
18 + return dict1 == dict2
19 +
20 +
21 +def should_ignore_layer(layer_name: Optional[str],
22 + ignore: Iterable[str]) -> bool:
23 + if layer_name is None:
24 + return False
25 +
26 + # layer_name = model.layers.0.self_attn.qkv_proj
27 + # proj_name = qkv_proj
28 + proj_name = layer_name.split(".")[-1]
29 +
30 + # Fused layers like gate_up_proj or qkv_proj will not be fused
31 + # in the safetensors checkpoint. So, we convert the name
32 + # from the fused version to unfused + check to make sure that
33 + # each shard of the fused layer has the same scheme.
34 + if proj_name in FUSED_LAYER_NAME_MAPPING:
35 + shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
36 +
37 + # Convert fused_name --> [shard_names]
38 + shard_names = [
39 + layer_name.replace(proj_name, shard_proj_name)
40 + for shard_proj_name in shard_proj_names
41 + ]
42 +
43 + # Layer should be ignored if shards are ignored.
44 + should_ignore_layer = None
45 + for shard_name in shard_names:
46 + should_ignore_shard = check_equal_or_regex_match(
47 + layer_name=shard_name, targets=ignore)
48 +
49 + # If shard_idx=0, set layer ignore to match shard.
50 + if should_ignore_layer is None:
51 + should_ignore_layer = should_ignore_shard
52 +
53 + # If shard_idx=1+ confirm scheme matches prior shards.
54 + elif should_ignore_shard != should_ignore_layer:
55 + raise ValueError(f"Found a different quantization schemes for "
56 + f"{shard_proj_names} in {layer_name}. vLLM "
57 + "requires all to use the same scheme.")
58 +
59 + # Unfused layers like down_proj and o_proj will match
60 + # the safetensors checkpoint already.
61 + else:
62 + should_ignore_layer = check_equal_or_regex_match(layer_name=layer_name,
63 + targets=ignore)
64 +
65 + assert should_ignore_layer is not None
66 + return should_ignore_layer
67 +
68 +
69 +def check_equal_or_regex_match(layer_name: str,
70 + targets: Iterable[str]) -> bool:
71 + """
72 + Checks whether a layer_name is exactly equal or a regex match for
73 + if target starts with 're:' to any target in list.
74 + """
75 + for target in targets:
76 + if _is_equal_or_regex_match(layer_name, target):
77 + return True
78 + return False
79 +
80 +
81 +def _is_equal_or_regex_match(value: str,
82 + target: str,
83 + check_contains: bool = False) -> bool:
84 + """
85 + Checks whether a value is exactly equal or a regex match for target
86 + if target starts with 're:'. If check_contains is set to True,
87 + additionally checks if the target string is contained within the value.
88 + """
89 +
90 + if target.startswith("re:"):
91 + pattern = target[3:]
92 + if re.match(pattern, value):
93 + return True
94 + elif check_contains:
95 + if target.lower() in value.lower():
96 + return True
97 + elif target == value:
98 + return True
99 + return False
vllm/model_executor/models/aria.py
+6 -5
@@ -13,8 +13,6 @@ from vllm.model_executor.layers.fused_moe import FusedMoE
13 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 RowParallelLinear)
15 from vllm.model_executor.layers.logits_processor import LogitsProcessor
16 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
17 - get_compressed_tensors_cache_scale)
16 from vllm.model_executor.layers.sampler import (SamplerOutput,
17 SamplingMetadata, get_sampler)
18 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
@@ -390,12 +388,15 @@ class AriaMoELMModel(LlamaModel):
388 # Models trained using ColossalAI may include these tensors in
389 # the checkpoint. Skip them.
390 continue
393 - if scale_name := get_compressed_tensors_cache_scale(name):
394 - # Loading kv cache scales for compressed-tensors quantization
391 + if (self.quant_config is not None and
392 + (scale_name := self.quant_config.get_cache_scale(name))):
393 + # Loading kv cache scales for quark and
394 + # compressed-tensors quantization
395 param = params_dict[scale_name]
396 weight_loader = getattr(param, "weight_loader",
397 default_weight_loader)
398 - loaded_weight = loaded_weight[0]
398 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
399 + loaded_weight[0])
400 weight_loader(param, loaded_weight)
401 loaded_params.add(scale_name)
402 continue
vllm/model_executor/models/commandr.py
+14
@@ -437,6 +437,20 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
437 params_dict = dict(self.named_parameters())
438 loaded_params: Set[str] = set()
439 for name, loaded_weight in weights:
440 +
441 + if (self.quant_config is not None and
442 + (scale_name := self.quant_config.get_cache_scale(name))):
443 + # Loading kv cache scales for quark and
444 + # compressed-tensors quantization
445 + param = params_dict[scale_name]
446 + weight_loader = getattr(param, "weight_loader",
447 + default_weight_loader)
448 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
449 + loaded_weight[0])
450 + weight_loader(param, loaded_weight)
451 + loaded_params.add(scale_name)
452 + continue
453 +
454 for param_name, shard_name, shard_id in stacked_params_mapping:
455 if shard_name not in name:
456 continue
vllm/model_executor/models/dbrx.py
+52 -21
@@ -83,7 +83,7 @@ class DbrxExperts(FusedMoE):
83
84 # Define custom weight loader for dbrx model
85 def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
86 - weight_name: str):
86 + weight_name: str, param_name: str):
87 tp_rank = get_tensor_model_parallel_rank()
88 param_data = param.data
89 shard_size = self.intermediate_size
@@ -91,25 +91,37 @@ class DbrxExperts(FusedMoE):
91 # DBRX uses GLU for each experts.
92 # GLU has 3 linear layers: w1, v1 and w2.
93 if weight_name.endswith("w1"):
94 - loaded_weight = torch.reshape(
95 - loaded_weight,
96 - [-1, self.intermediate_size * self.tp_size, self.d_model],
97 - )
98 - param_data[:, 0:shard_size, :] = loaded_weight[:, shard, :]
94 + if param_name.endswith("weight"):
95 + loaded_weight = torch.reshape(
96 + loaded_weight,
97 + [-1, self.intermediate_size * self.tp_size, self.d_model],
98 + )
99 + param_data[:, 0:shard_size, :] = loaded_weight[:, shard, :]
100 + elif param_name.endswith("weight_scale"):
101 + param_data[:, 0] = loaded_weight
102 + else:
103 + param_data = loaded_weight
104 if weight_name.endswith("v1"):
100 - loaded_weight = torch.reshape(
101 - loaded_weight,
102 - [-1, self.intermediate_size * self.tp_size, self.d_model],
103 - )
104 - param_data[:,
105 - shard_size:2 * shard_size, :] = loaded_weight[:,
106 - shard, :]
105 + if param_name.endswith("weight"):
106 + loaded_weight = torch.reshape(
107 + loaded_weight,
108 + [-1, self.intermediate_size * self.tp_size, self.d_model],
109 + )
110 + param_data[:, shard_size:2 *
111 + shard_size, :] = loaded_weight[:, shard, :]
112 + elif param_name.endswith("weight_scale"):
113 + param_data[:, 1] = loaded_weight
114 + else:
115 + param_data[:] = loaded_weight
116 if weight_name.endswith("w2"):
108 - loaded_weight = torch.reshape(
109 - loaded_weight,
110 - [-1, self.intermediate_size * self.tp_size, self.d_model],
111 - ).transpose(1, 2)
112 - param_data[:] = loaded_weight[:, :, shard]
117 + if param_name.endswith("weight"):
118 + loaded_weight = torch.reshape(
119 + loaded_weight,
120 + [-1, self.intermediate_size * self.tp_size, self.d_model],
121 + ).transpose(1, 2)
122 + param_data[:] = loaded_weight[:, :, shard]
123 + else:
124 + param_data[:] = loaded_weight
125
126
127 class DbrxMoE(nn.Module):
@@ -430,14 +442,29 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
442
443 def load_weights(self, weights: Iterable[Tuple[str,
444 torch.Tensor]]) -> Set[str]:
433 -
445 expert_params_mapping = [(
435 - "w13_weight" if weight_name in ["w1", "v1"] else "w2_weight",
446 + "w13" if weight_name in ["w1", "v1"] else "w2",
447 f"mlp.{weight_name}",
448 ) for weight_name in ["w1", "v1", "w2"]]
449 params_dict = dict(self.named_parameters(remove_duplicate=False))
450 loaded_params: Set[str] = set()
451 +
452 for name, loaded_weight in weights:
453 + if (self.quant_config is not None and
454 + (scale_name := self.quant_config.get_cache_scale(name))):
455 + # Loading kv cache scales for quark and
456 + # compressed-tensors quantization
457 + param = params_dict[scale_name]
458 + weight_loader = getattr(param, "weight_loader",
459 + default_weight_loader)
460 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
461 + loaded_weight[0])
462 + weight_loader(param, loaded_weight)
463 + loaded_params.add(scale_name)
464 + continue
465 +
466 + if name.endswith(("w1", "w2", "v1")):
467 + name = name + "_weight"
468 for param_name, weight_name in expert_params_mapping:
469 if weight_name not in name:
470 continue
@@ -446,8 +473,9 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
473 continue
474 param = params_dict[name]
475 weight_loader = param.weight_loader
449 - weight_loader(param, loaded_weight, weight_name)
476 + weight_loader(param, loaded_weight, weight_name, name)
477 break
478 +
479 else:
480 # Remapping the name of FP8 kv-scale.
481 name = maybe_remap_kv_scale_name(name, params_dict)
@@ -456,6 +484,9 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
484
485 if is_pp_missing_parameter(name, self):
486 continue
487 + name = maybe_remap_kv_scale_name(name, params_dict)
488 + if name is None:
489 + continue
490 param = params_dict[name]
491 weight_loader = getattr(param, "weight_loader",
492 default_weight_loader)
vllm/model_executor/models/exaone.py
+7 -5
@@ -39,8 +39,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39 RowParallelLinear)
40 from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 from vllm.model_executor.layers.quantization import QuantizationConfig
42 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43 - get_compressed_tensors_cache_scale)
42 from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -439,6 +437,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
437
438 self.config = config
439 self.lora_config = lora_config
440 + self.quant_config = quant_config
441
442 self.transformer = ExaoneModel(
443 vllm_config=vllm_config,
@@ -532,12 +531,15 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
531 # processed with quantization, LoRA, fine-tuning, etc.
532 if self.config.tie_word_embeddings and "lm_head.weight" in name:
533 continue
535 - if scale_name := get_compressed_tensors_cache_scale(name):
536 - # Loading kv cache scales for compressed-tensors quantization
534 + if (self.quant_config is not None and
535 + (scale_name := self.quant_config.get_cache_scale(name))):
536 + # Loading kv cache scales for quark and
537 + # compressed-tensors quantization
538 param = params_dict[scale_name]
539 weight_loader = getattr(param, "weight_loader",
540 default_weight_loader)
540 - loaded_weight = loaded_weight[0]
541 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
542 + loaded_weight[0])
543 weight_loader(param, loaded_weight)
544 loaded_params.add(scale_name)
545 continue
vllm/model_executor/models/gemma2.py
+3 -3
@@ -31,8 +31,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31 RowParallelLinear)
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
35 - get_compressed_tensors_cache_scale)
34 from vllm.model_executor.layers.rotary_embedding import get_rope
35 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -254,6 +252,7 @@ class Gemma2Model(nn.Module):
252 cache_config = vllm_config.cache_config
253 quant_config = vllm_config.quant_config
254 self.config = config
255 + self.quant_config = quant_config
256
257 self.embed_tokens = VocabParallelEmbedding(
258 config.vocab_size,
@@ -329,7 +328,8 @@ class Gemma2Model(nn.Module):
328 params_dict = dict(self.named_parameters())
329 loaded_params: Set[str] = set()
330 for name, loaded_weight in weights:
332 - if scale_name := get_compressed_tensors_cache_scale(name):
331 + if (self.quant_config is not None and
332 + (scale_name := self.quant_config.get_cache_scale(name))):
333 # Loading kv cache scales for compressed-tensors quantization
334 param = params_dict[scale_name]
335 weight_loader = getattr(param, "weight_loader",
vllm/model_executor/models/gpt_j.py
+14
@@ -313,6 +313,20 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
313 for name, loaded_weight in weights:
314 if "attn.bias" in name or "attn.masked_bias" in name:
315 continue
316 +
317 + if (self.quant_config is not None and
318 + (scale_name := self.quant_config.get_cache_scale(name))):
319 + # Loading kv cache scales for quark and
320 + # compressed-tensors quantization
321 + param = params_dict[scale_name]
322 + weight_loader = getattr(param, "weight_loader",
323 + default_weight_loader)
324 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
325 + loaded_weight[0])
326 + weight_loader(param, loaded_weight)
327 + loaded_params.add(scale_name)
328 + continue
329 +
330 for (param_name, weight_name, shard_id) in stacked_params_mapping:
331 if weight_name not in name:
332 continue
vllm/model_executor/models/granite.py
+7 -5
@@ -39,8 +39,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 from vllm.model_executor.layers.quantization.base_config import (
41 QuantizationConfig)
42 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43 - get_compressed_tensors_cache_scale)
42 from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -371,6 +369,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
369
370 self.config = config
371 self.lora_config = lora_config
372 + self.quant_config = quant_config
373
374 self.model = GraniteModel(vllm_config=vllm_config,
375 prefix=maybe_prefix(prefix, "model"))
@@ -474,12 +473,15 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
473 # processed with quantization, LoRA, fine-tuning, etc.
474 if self.config.tie_word_embeddings and "lm_head.weight" in name:
475 continue
477 - if scale_name := get_compressed_tensors_cache_scale(name):
478 - # Loading kv cache scales for compressed-tensors quantization
476 + if (self.quant_config is not None and
477 + (scale_name := self.quant_config.get_cache_scale(name))):
478 + # Loading kv cache scales for quark and
479 + # compressed-tensors quantization
480 param = params_dict[scale_name]
481 weight_loader = getattr(param, "weight_loader",
482 default_weight_loader)
482 - loaded_weight = loaded_weight[0]
483 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
484 + loaded_weight[0])
485 weight_loader(param, loaded_weight)
486 loaded_params.add(scale_name)
487 continue
vllm/model_executor/models/llama.py
+7 -5
@@ -38,8 +38,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
38 RowParallelLinear)
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 from vllm.model_executor.layers.quantization import QuantizationConfig
41 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
42 - get_compressed_tensors_cache_scale)
41 from vllm.model_executor.layers.rotary_embedding import get_rope
42 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -306,6 +304,7 @@ class LlamaModel(nn.Module):
304 lora_config = vllm_config.lora_config
305
306 self.config = config
307 + self.quant_config = quant_config
308 self.padding_idx = config.pad_token_id
309 lora_vocab = (lora_config.lora_extra_vocab_size *
310 (lora_config.max_loras or 1)) if lora_config else 0
@@ -396,12 +395,15 @@ class LlamaModel(nn.Module):
395 # Models trained using ColossalAI may include these tensors in
396 # the checkpoint. Skip them.
397 continue
399 - if scale_name := get_compressed_tensors_cache_scale(name):
400 - # Loading kv cache scales for compressed-tensors quantization
398 + if (self.quant_config is not None and
399 + (scale_name := self.quant_config.get_cache_scale(name))):
400 + # Loading kv cache scales for quark and
401 + # compressed-tensors quantization
402 param = params_dict[scale_name]
403 weight_loader = getattr(param, "weight_loader",
404 default_weight_loader)
404 - loaded_weight = loaded_weight[0]
405 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
406 + loaded_weight[0])
407 weight_loader(param, loaded_weight)
408 loaded_params.add(scale_name)
409 continue
vllm/model_executor/models/mixtral.py
+14
@@ -347,6 +347,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
347 lora_config = vllm_config.lora_config
348 self.config = config
349 self.lora_config = lora_config
350 + self.quant_config = quant_config
351
352 self.model = MixtralModel(vllm_config=vllm_config,
353 prefix=maybe_prefix(prefix, "model"))
@@ -428,6 +429,19 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
429 if "rotary_emb.inv_freq" in name:
430 continue
431
432 + if (self.quant_config is not None and
433 + (scale_name := self.quant_config.get_cache_scale(name))):
434 + # Loading kv cache scales for quark and
435 + # compressed-tensors quantization
436 + param = params_dict[scale_name]
437 + weight_loader = getattr(param, "weight_loader",
438 + default_weight_loader)
439 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
440 + loaded_weight[0])
441 + weight_loader(param, loaded_weight)
442 + loaded_params.add(scale_name)
443 + continue
444 +
445 for (param_name, weight_name, shard_id) in stacked_params_mapping:
446 if weight_name not in name:
447 continue
vllm/model_executor/models/mllama.py
+13
@@ -1116,6 +1116,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1116 super().__init__()
1117 config = vllm_config.model_config.hf_config
1118 quant_config = vllm_config.quant_config
1119 + self.quant_config = quant_config
1120 self.vocab_size = config.text_config.vocab_size
1121 self.hidden_size = config.text_config.hidden_size
1122 self.max_num_tiles = config.vision_config.max_num_tiles
@@ -1429,6 +1430,18 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1430 name = name.replace('patch_embedding.weight',
1431 'patch_embedding._linear.weight')
1432 loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
1433 + if (self.quant_config is not None and
1434 + (scale_name := self.quant_config.get_cache_scale(name))):
1435 + # Loading kv cache scales for quark and
1436 + # compressed-tensors quantization
1437 + param = params_dict[scale_name]
1438 + weight_loader = getattr(param, "weight_loader",
1439 + default_weight_loader)
1440 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
1441 + loaded_weight[0])
1442 + weight_loader(param, loaded_weight)
1443 + updated_params.add(scale_name)
1444 + continue
1445 for (param_name, weight_name, shard_id) in stacked_params_mapping:
1446 if weight_name not in name:
1447 continue
vllm/model_executor/models/nemotron.py
+13
@@ -405,6 +405,7 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
405
406 self.config = config
407 self.lora_config = lora_config
408 + self.quant_config = quant_config
409
410 self.model = NemotronModel(vllm_config=vllm_config,
411 prefix=maybe_prefix(prefix, "model"))
@@ -489,6 +490,18 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
490 # Models trained using ColossalAI may include these tensors in
491 # the checkpoint. Skip them.
492 continue
493 + if (self.quant_config is not None and
494 + (scale_name := self.quant_config.get_cache_scale(name))):
495 + # Loading kv cache scales for quark and
496 + # compressed-tensors quantization
497 + param = params_dict[scale_name]
498 + weight_loader = getattr(param, "weight_loader",
499 + default_weight_loader)
500 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
501 + loaded_weight[0])
502 + weight_loader(param, loaded_weight)
503 + loaded_params.add(scale_name)
504 + continue
505 for (param_name, weight_name, shard_id) in stacked_params_mapping:
506 if weight_name not in name:
507 continue
vllm/model_executor/models/phimoe.py
+14
@@ -546,6 +546,7 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
546 lora_config = vllm_config.lora_config
547 self.config = config
548 self.lora_config = lora_config
549 + self.quant_config = vllm_config.quant_config
550
551 self.model = PhiMoEModel(vllm_config=vllm_config,
552 prefix=maybe_prefix(prefix, "model"))
@@ -623,6 +624,19 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
624 if "rotary_emb.inv_freq" in name:
625 continue
626
627 + if (self.quant_config is not None and
628 + (scale_name := self.quant_config.get_cache_scale(name))):
629 + # Loading kv cache scales for quark and
630 + # compressed-tensors quantization
631 + param = params_dict[scale_name]
632 + weight_loader = getattr(param, "weight_loader",
633 + default_weight_loader)
634 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
635 + loaded_weight[0])
636 + weight_loader(param, loaded_weight)
637 + loaded_params.add(scale_name)
638 + continue
639 +
640 for param_name, weight_name, shard_id in stacked_params_mapping:
641 if weight_name not in name:
642 continue
vllm/model_executor/models/qwen2.py
+13
@@ -279,6 +279,7 @@ class Qwen2Model(nn.Module):
279 ))
280
281 self.config = config
282 + self.quant_config = quant_config
283 self.padding_idx = config.pad_token_id
284 self.vocab_size = config.vocab_size
285
@@ -364,6 +365,18 @@ class Qwen2Model(nn.Module):
365 for name, loaded_weight in weights:
366 if "rotary_emb.inv_freq" in name:
367 continue
368 + if (self.quant_config is not None and
369 + (scale_name := self.quant_config.get_cache_scale(name))):
370 + # Loading kv cache scales for quark and
371 + # compressed-tensors quantization
372 + param = params_dict[scale_name]
373 + weight_loader = getattr(param, "weight_loader",
374 + default_weight_loader)
375 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
376 + loaded_weight[0])
377 + weight_loader(param, loaded_weight)
378 + loaded_params.add(scale_name)
379 + continue
380 for (param_name, weight_name, shard_id) in stacked_params_mapping:
381 if weight_name not in name:
382 continue
vllm/model_executor/models/solar.py
+7 -5
@@ -39,8 +39,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39 RowParallelLinear)
40 from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 from vllm.model_executor.layers.quantization import QuantizationConfig
42 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43 - get_compressed_tensors_cache_scale)
42 from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -409,6 +407,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
407 lora_config = vllm_config.lora_config
408 self.config = config
409 self.lora_config = lora_config
410 + self.quant_config = quant_config
411
412 self.model = SolarModel(
413 vllm_config=vllm_config,
@@ -491,12 +490,15 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
490 # Models trained using ColossalAI may include these tensors in
491 # the checkpoint. Skip them.
492 continue
494 - if scale_name := get_compressed_tensors_cache_scale(name):
495 - # Loading kv cache scales for compressed-tensors quantization
493 + if (self.quant_config is not None and
494 + (scale_name := self.quant_config.get_cache_scale(name))):
495 + # Loading kv cache scales for quark and
496 + # compressed-tensors quantization
497 param = params_dict[scale_name]
498 weight_loader = getattr(param, "weight_loader",
499 default_weight_loader)
499 - loaded_weight = loaded_weight[0]
500 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
501 + loaded_weight[0])
502 weight_loader(param, loaded_weight)
503 loaded_params.add(scale_name)
504 continue
vllm/model_executor/parameter.py
+7 -1
@@ -56,8 +56,14 @@ class BasevLLMParameter(Parameter):
56 def weight_loader(self):
57 return self._weight_loader
58
59 + def _is_1d_and_scalar(self, loaded_weight: torch.Tensor):
60 + cond1 = self.data.ndim == 1 and self.data.numel() == 1
61 + cond2 = loaded_weight.ndim == 0 and loaded_weight.numel() == 1
62 + return (cond1 and cond2)
63 +
64 def _assert_and_load(self, loaded_weight: torch.Tensor):
60 - assert self.data.shape == loaded_weight.shape
65 + assert (self.data.shape == loaded_weight.shape
66 + or self._is_1d_and_scalar(loaded_weight))
67 self.data.copy_(loaded_weight)
68
69 def load_column_parallel_weight(self, loaded_weight: torch.Tensor):
vllm/platforms/rocm.py
+1 -1
@@ -70,7 +70,7 @@ class RocmPlatform(Platform):
70
71 supported_quantization: list[str] = [
72 "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
73 - "fbgemm_fp8", "gguf"
73 + "fbgemm_fp8", "gguf", "quark"
74 ]
75
76 @classmethod