@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 082ecd80d5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Bugfix ] Fix AutoFP8 fp8 marlin (#6609)

Robert Shaw committed Jul 20, 2024 at 19:25 UTC 082ecd80d58c6604f44c0196cb9db5bc4befd6d7
1 file changed +2 -1
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+2 -1
@@ -76,7 +76,8 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
76 # WEIGHT SCALES
77 # Currently Marlin doesn't support per-tensor scales, so we
78 # expand it to channelwise
79 - is_channelwise = layer.weight_scale.shape[0] == part_size_n
79 + is_channelwise = (len(layer.weight_scale.shape) > 0
80 + and layer.weight_scale.shape[0] == part_size_n)
81 if is_channelwise:
82 scales = layer.weight_scale
83 else: