A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Fix empty (nullptr) channelwise scales when loading wNa16 using compressed tensors (#6798)
Lucas Wilkinson committed
Jul 25, 2024 at 18:05 UTC
cd7edc4e8726d4b87e121f9ec671ecb6dd0c45d6
1 file changed
+8
-3
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+8
-3
@@ -55,7 +55,12 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
55
output_size_per_partition = sum(output_partition_sizes)
56
57
# If group_size is -1, we are in channelwise case.
58
- group_size = input_size if self.group_size == -1 else self.group_size
58
+ channelwise = (self.group_size == -1)
59
+ group_size = input_size if channelwise else self.group_size
60
+ row_parallel = (input_size != input_size_per_partition)
61
+ # In the case of channelwise quantization, we need to replicate the
62
+ # scales across all gpus.
63
+ partition_scales = (row_parallel and not channelwise)
64
65
verify_marlin_supports_shape(
66
output_size_per_partition=output_size_per_partition,
@@ -66,8 +71,8 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
71
weight_scale_dim = None
72
scales_and_zp_size = input_size // group_size
73
69
- if (input_size != input_size_per_partition
70
- and self.group_size is not None):
74
+ if partition_scales:
75
+ assert input_size_per_partition % group_size == 0
76
weight_scale_dim = 1
77
scales_and_zp_size = input_size_per_partition // group_size
78