A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ Bugfix ] Fix AutoFP8 fp8 marlin (#6609)
Robert Shaw committed
Jul 20, 2024 at 19:25 UTC
082ecd80d58c6604f44c0196cb9db5bc4befd6d7
1 file changed
+2
-1
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+2
-1
@@ -76,7 +76,8 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
76
# WEIGHT SCALES
77
# Currently Marlin doesn't support per-tensor scales, so we
78
# expand it to channelwise
79
- is_channelwise = layer.weight_scale.shape[0] == part_size_n
79
+ is_channelwise = (len(layer.weight_scale.shape) > 0
80
+ and layer.weight_scale.shape[0] == part_size_n)
81
if is_channelwise:
82
scales = layer.weight_scale
83
else: