A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] Avoid Warnings in SparseML Activation Quantization (#5120)
Robert Shaw committed
May 30, 2024 at 17:04 UTC
b35be5403f3cf8631aefe02a35d97013657e2e47
1 file changed
+20
-9
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+20
-9
@@ -89,23 +89,34 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
89
requires_grad=False)
90
91
layer.register_parameter("weight", weight)
92
- set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
93
-
94
- set_weight_attrs(weight, {"weight_loader": weight_loader})
95
-
92
+ set_weight_attrs(weight, {
93
+ "weight_loader": weight_loader,
94
+ "input_dim": 1,
95
+ "output_dim": 0,
96
+ })
97
layer.register_parameter("input_scale", input_scale)
97
- set_weight_attrs(input_scale, {"weight_loader": weight_loader})
98
+ set_weight_attrs(input_scale, {
99
+ "weight_loader": weight_loader,
100
+ "ignore_warning": True,
101
+ })
102
layer.register_parameter("input_zero_point", input_zero_point)
99
- set_weight_attrs(input_zero_point, {"weight_loader": weight_loader})
103
+ set_weight_attrs(input_zero_point, {
104
+ "weight_loader": weight_loader,
105
+ "ignore_warning": True,
106
+ })
107
layer.register_parameter("weight_scale", weight_scale)
101
- set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
108
set_weight_attrs(
109
weight_scale, {
110
+ "weight_loader": weight_loader,
111
"shard_splitter": self.scales_shard_splitter,
105
- "logical_widths": output_partition_sizes
112
+ "logical_widths": output_partition_sizes,
113
+ "ignore_warning": True,
114
})
115
layer.register_parameter("weight_zero_point", weight_zero_point)
108
- set_weight_attrs(weight_zero_point, {"weight_loader": weight_loader})
116
+ set_weight_attrs(weight_zero_point, {
117
+ "weight_loader": weight_loader,
118
+ "ignore_warning": True
119
+ })
120
121
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
122
weight = layer.weight