| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Iterable, Mapping |
| 5 | from types import MappingProxyType |
| 6 | from typing import Any |
| 7 | |
| 8 | import torch |
| 9 | |
| 10 | from vllm.model_executor.layers.quantization.utils.config_utils import ( |
| 11 | find_matching_patterns, |
| 12 | ) |
| 13 | |
| 14 | QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None |
| 15 | |
| 16 | |
| 17 | def deep_compare(dict1: Any, dict2: Any) -> bool: |
| 18 | if type(dict1) is not type(dict2): |
| 19 | return False |
| 20 | if isinstance(dict1, dict): |
| 21 | if dict1.keys() != dict2.keys(): |
| 22 | return False |
| 23 | return all(deep_compare(dict1[k], dict2[k]) for k in dict1) |
| 24 | elif isinstance(dict1, list): |
| 25 | # `dict1` may be a list of dict. |
| 26 | return all(deep_compare(dict1[i], dict2[i]) for i in range(len(dict1))) |
| 27 | else: |
| 28 | return dict1 == dict2 |
| 29 | |
| 30 | |
| 31 | def should_ignore_layer( |
| 32 | layer_name: str | None, |
| 33 | ignore: Iterable[str], |
| 34 | fused_mapping: Mapping[str, list[str]] = MappingProxyType({}), |
| 35 | *, |
| 36 | check_children: bool = False, |
| 37 | ) -> bool: |
| 38 | if layer_name is None: |
| 39 | return False |
| 40 | |
| 41 | # MoE layers are currently all-or-nothing: if any child is ignored, |
| 42 | # the parent layer must be ignored as well. For example, the |
| 43 | # amd/GLM-5.2-MXFP4 config ignores children like |
| 44 | # model.layers.78.mlp.experts.*.down_proj, while the layer checked |
| 45 | # here is the parent model.layers.N.mlp.experts. |
| 46 | # See: |
| 47 | # https://huggingface.co/amd/GLM-5.2-MXFP4/blob/main/config.json#L793-L795 |
| 48 | if check_children and any( |
| 49 | target == layer_name or target.startswith(layer_name + ".") |
| 50 | for target in ignore |
| 51 | if not target.startswith("re:") |
| 52 | ): |
| 53 | return True |
| 54 | |
| 55 | # A direct fused-layer pattern takes precedence over expansion. For |
| 56 | # model.layers.0.self_attn.qkv_proj, |
| 57 | # ignore=["re:.*qkv_proj.*"] yields [{"re:.*qkv_proj.*"}]. In contrast, |
| 58 | # ignore=["re:.*[qkv]_proj"] yields one matching set per expanded shard. |
| 59 | per_shard_matches = find_matching_patterns(layer_name, ignore, fused_mapping) |
| 60 | shards_ignored = [len(matches) > 0 for matches in per_shard_matches] |
| 61 | if any(shards_ignored) and not all(shards_ignored): |
| 62 | raise ValueError( |
| 63 | f"Found different quantization schemes for the shards of " |
| 64 | f"{layer_name}. vLLM requires all to use the same scheme." |
| 65 | ) |
| 66 | return all(shards_ignored) |
| 67 | |
| 68 | |
| 69 | def parse_w4a16_int4_weight_config( |
| 70 | weight_config: Mapping[str, Any], |
| 71 | ) -> tuple[int, bool]: |
| 72 | """Parse required W4A16 INT4/UINT4 weight fields from Quark config.""" |
| 73 | if "group_size" not in weight_config: |
| 74 | raise ValueError( |
| 75 | "Quark W4A16 INT4/UINT4 configs must specify weight.group_size" |
| 76 | ) |
| 77 | if "symmetric" not in weight_config: |
| 78 | raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric") |
| 79 | |
| 80 | group_size = weight_config["group_size"] |
| 81 | is_symmetric = weight_config["symmetric"] |
| 82 | if not isinstance(group_size, int) or group_size <= 0: |
| 83 | raise ValueError( |
| 84 | f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}" |
| 85 | ) |
| 86 | if not isinstance(is_symmetric, bool): |
| 87 | raise ValueError( |
| 88 | f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}" |
| 89 | ) |
| 90 | return group_size, is_symmetric |
| 91 | |
| 92 | |
| 93 | def canonicalize_quark_packed_int4( |
| 94 | packed_weight: torch.Tensor, |
| 95 | *, |
| 96 | pack_reorder: bool, |
| 97 | is_symmetric: bool, |
| 98 | pack_factor: int = 8, |
| 99 | ) -> torch.Tensor: |
| 100 | """Convert Quark export nibble layout to AWQ checkpoint layout.""" |
| 101 | from vllm.model_executor.layers.quantization.auto_awq import ( |
| 102 | _REVERSE_AWQ_PACK_ORDER, |
| 103 | ) |
| 104 | |
| 105 | if pack_reorder: |
| 106 | source_order = torch.tensor( |
| 107 | _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32 |
| 108 | ) |
| 109 | else: |
| 110 | source_order = torch.arange( |
| 111 | pack_factor, device=packed_weight.device, dtype=torch.int32 |
| 112 | ) |
| 113 | target_order = torch.tensor( |
| 114 | _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32 |
| 115 | ) |
| 116 | source_shifts = source_order * 4 |
| 117 | target_shifts = target_order * 4 |
| 118 | |
| 119 | values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF |
| 120 | if is_symmetric: |
| 121 | values = values ^ 0x8 |
| 122 | packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1) |
| 123 | return packed.to(torch.int32) |
| 124 | |
| 125 | |
| 126 | # utility for tensor dims > 2 cases |
| 127 | def quark_quantize_weight_to_mxfp4(w: torch.Tensor): |
| 128 | assert w.dtype == torch.bfloat16, ( |
| 129 | "Quark dynamic quantization is supported only for fp16 weights and only to MXF4" |
| 130 | ) |
| 131 | |
| 132 | from aiter.ops.triton.quant import dynamic_mxfp4_quant |
| 133 | |
| 134 | *dims, d = w.shape |
| 135 | w, w_scales = dynamic_mxfp4_quant(w.reshape(-1, d)) |
| 136 | return w.view(*dims, d // 2), w_scales.view(*dims, d // 32) |