@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b35be5403f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] Avoid Warnings in SparseML Activation Quantization (#5120)

Robert Shaw committed May 30, 2024 at 17:04 UTC b35be5403f3cf8631aefe02a35d97013657e2e47
1 file changed +20 -9
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+20 -9
@@ -89,23 +89,34 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
89 requires_grad=False)
90
91 layer.register_parameter("weight", weight)
92 - set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
93 -
94 - set_weight_attrs(weight, {"weight_loader": weight_loader})
95 -
92 + set_weight_attrs(weight, {
93 + "weight_loader": weight_loader,
94 + "input_dim": 1,
95 + "output_dim": 0,
96 + })
97 layer.register_parameter("input_scale", input_scale)
97 - set_weight_attrs(input_scale, {"weight_loader": weight_loader})
98 + set_weight_attrs(input_scale, {
99 + "weight_loader": weight_loader,
100 + "ignore_warning": True,
101 + })
102 layer.register_parameter("input_zero_point", input_zero_point)
99 - set_weight_attrs(input_zero_point, {"weight_loader": weight_loader})
103 + set_weight_attrs(input_zero_point, {
104 + "weight_loader": weight_loader,
105 + "ignore_warning": True,
106 + })
107 layer.register_parameter("weight_scale", weight_scale)
101 - set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
108 set_weight_attrs(
109 weight_scale, {
110 + "weight_loader": weight_loader,
111 "shard_splitter": self.scales_shard_splitter,
105 - "logical_widths": output_partition_sizes
112 + "logical_widths": output_partition_sizes,
113 + "ignore_warning": True,
114 })
115 layer.register_parameter("weight_zero_point", weight_zero_point)
108 - set_weight_attrs(weight_zero_point, {"weight_loader": weight_loader})
116 + set_weight_attrs(weight_zero_point, {
117 + "weight_loader": weight_loader,
118 + "ignore_warning": True
119 + })
120
121 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
122 weight = layer.weight