A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Bugfix] Support MLA for CompressedTensorsWNA16 (#13725)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Feb 25, 2025 at 01:10 UTC
18e505930d789a2ad57c8a048ff7d84c025530bd
1 file changed
+7
-7
vllm/attention/backends/mla/common.py
+7
-7
@@ -1130,13 +1130,13 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
1130
)
1131
1132
def get_layer_weight(layer):
1133
- if hasattr(layer, "weight"):
1134
- return layer.weight
1135
- elif hasattr(layer, "qweight"):
1136
- return layer.qweight
1137
- else:
1138
- raise AttributeError(
1139
- f"Layer '{layer}' has neither weight nor qweight")
1133
+ WEIGHT_NAMES = ("weight", "qweight", "weight_packed")
1134
+ for attr in WEIGHT_NAMES:
1135
+ if hasattr(layer, attr):
1136
+ return getattr(layer, attr)
1137
+ raise AttributeError(
1138
+ f"Layer '{layer}' has no recognized weight attribute:"
1139
+ f" {WEIGHT_NAMES}.")
1140
1141
def get_and_maybe_dequant_weights(layer: LinearBase):
1142
if not isinstance(layer.quant_method, UnquantizedLinearMethod):