@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 15985680e2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[ Misc ] Rs/compressed tensors cleanup (#5432)

Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: Dipika Sikka <dipikasikka1@gmail.com>

Robert Shaw committed Jun 14, 2024 at 13:01 UTC 15985680e2278610e873cc07ec72fa514ace72e9
4 files changed +21 -36
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+1 -1
@@ -26,7 +26,7 @@ class CompressedTensorsConfig(QuantizationConfig):
26 return []
27
28 def get_supported_act_dtypes(cls) -> List[torch.dtype]:
29 - return [torch.float16]
29 + return [torch.float16, torch.bfloat16]
30
31 # Need to figure it out
32 def get_min_capability(self) -> int:
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16.py
+12 -9
@@ -64,10 +64,9 @@ class CompressedTensorsW4A16(CompressedTensorsScheme):
64 "input_dim": 1,
65 "output_dim": 0,
66 "packed_dim": 1,
67 - "pack_factor": pack_factor
67 + "pack_factor": pack_factor,
68 + "weight_loader": weight_loader
69 })
69 - set_weight_attrs(weight, {"weight_loader": weight_loader})
70 -
70 layer.register_parameter("weight_packed", weight)
71
72 weight_scale = Parameter(
@@ -79,11 +78,12 @@ class CompressedTensorsW4A16(CompressedTensorsScheme):
78 requires_grad=False,
79 )
80
82 - set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
83 - set_weight_attrs(weight_scale, {
84 - "input_dim": weight_scale_dim,
85 - "output_dim": 0
86 - })
81 + set_weight_attrs(
82 + weight_scale, {
83 + "weight_loader": weight_loader,
84 + "input_dim": weight_scale_dim,
85 + "output_dim": 0
86 + })
87 layer.register_parameter("weight_scale", weight_scale)
88
89 # A 2D array defining the original shape of the weights
@@ -92,7 +92,10 @@ class CompressedTensorsW4A16(CompressedTensorsScheme):
92 requires_grad=False)
93
94 layer.register_parameter("weight_shape", weight_shape)
95 - set_weight_attrs(weight_shape, {"weight_loader": weight_loader})
95 + set_weight_attrs(weight_shape, {
96 + "weight_loader": weight_loader,
97 + "ignore_warning": True,
98 + })
99
100 layer.input_size_per_partition = input_size_per_partition
101 layer.output_size_per_partition = output_size_per_partition
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py
+8 -10
@@ -48,9 +48,6 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
48 weight_scale_dim = sum(
49 output_partition_sizes) if is_tensor_partitioned else 1
50
51 - weight_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
52 - requires_grad=False)
53 -
51 weight_scale = Parameter(torch.empty(weight_scale_dim,
52 dtype=torch.float32),
53 requires_grad=False)
@@ -61,21 +58,22 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
58 requires_grad=False)
59
60 layer.register_parameter("weight", weight)
64 - set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
65 - set_weight_attrs(weight, {"weight_loader": weight_loader})
66 - set_weight_attrs(weight, {"logical_widths": output_partition_sizes})
61 + set_weight_attrs(
62 + weight, {
63 + "input_dim": 1,
64 + "output_dim": 0,
65 + "weight_loader": weight_loader,
66 + "logical_widths": output_partition_sizes
67 + })
68
69 layer.register_parameter("weight_scale", weight_scale)
69 - set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
70 set_weight_attrs(
71 weight_scale, {
72 + "weight_loader": weight_loader,
73 "shard_splitter": self.scales_shard_splitter,
74 "logical_widths": output_partition_sizes
75 })
76
76 - layer.register_parameter("weight_zero_point", weight_zero_point)
77 - set_weight_attrs(weight_zero_point, {"weight_loader": weight_loader})
78 -
77 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
78 weight = layer.weight
79 weight_scale = layer.weight_scale
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
-16
@@ -39,22 +39,16 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
39 params_dtype: torch.dtype, weight_loader: Callable,
40 **kwargs):
41
42 - # TODO: remove zero_point parameters once the configs given remove them
43 -
42 is_tensor_partitioned = len(output_partition_sizes) != 1
43 weight_scale_dim = sum(
44 output_partition_sizes) if is_tensor_partitioned else 1
45
46 input_scale = Parameter(torch.empty(1, dtype=torch.float32),
47 requires_grad=False)
50 - input_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
51 - requires_grad=False)
48
49 weight_scale = Parameter(torch.empty(weight_scale_dim,
50 dtype=torch.float32),
51 requires_grad=False)
56 - weight_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
57 - requires_grad=False)
52
53 weight = Parameter(torch.empty(sum(output_partition_sizes),
54 input_size_per_partition,
@@ -72,11 +66,6 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
66 "weight_loader": weight_loader,
67 "ignore_warning": True,
68 })
75 - layer.register_parameter("input_zero_point", input_zero_point)
76 - set_weight_attrs(input_zero_point, {
77 - "weight_loader": weight_loader,
78 - "ignore_warning": True,
79 - })
69 layer.register_parameter("weight_scale", weight_scale)
70 set_weight_attrs(
71 weight_scale, {
@@ -85,11 +74,6 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
74 "logical_widths": output_partition_sizes,
75 "ignore_warning": True,
76 })
88 - layer.register_parameter("weight_zero_point", weight_zero_point)
89 - set_weight_attrs(weight_zero_point, {
90 - "weight_loader": weight_loader,
91 - "ignore_warning": True
92 - })
77
78 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
79 weight = layer.weight