@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cd7edc4e87
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix empty (nullptr) channelwise scales when loading wNa16 using compressed tensors (#6798)

Lucas Wilkinson committed Jul 25, 2024 at 18:05 UTC cd7edc4e8726d4b87e121f9ec671ecb6dd0c45d6
1 file changed +8 -3
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+8 -3
@@ -55,7 +55,12 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
55 output_size_per_partition = sum(output_partition_sizes)
56
57 # If group_size is -1, we are in channelwise case.
58 - group_size = input_size if self.group_size == -1 else self.group_size
58 + channelwise = (self.group_size == -1)
59 + group_size = input_size if channelwise else self.group_size
60 + row_parallel = (input_size != input_size_per_partition)
61 + # In the case of channelwise quantization, we need to replicate the
62 + # scales across all gpus.
63 + partition_scales = (row_parallel and not channelwise)
64
65 verify_marlin_supports_shape(
66 output_size_per_partition=output_size_per_partition,
@@ -66,8 +71,8 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
71 weight_scale_dim = None
72 scales_and_zp_size = input_size // group_size
73
69 - if (input_size != input_size_per_partition
70 - and self.group_size is not None):
74 + if partition_scales:
75 + assert input_size_per_partition % group_size == 0
76 weight_scale_dim = 1
77 scales_and_zp_size = input_size_per_partition // group_size
78