A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[ Misc ] Rs/compressed tensors cleanup (#5432)
Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: Dipika Sikka <dipikasikka1@gmail.com>
Robert Shaw committed
Jun 14, 2024 at 13:01 UTC
15985680e2278610e873cc07ec72fa514ace72e9
4 files changed
+21
-36
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+1
-1
@@ -26,7 +26,7 @@ class CompressedTensorsConfig(QuantizationConfig):
26
return []
27
28
def get_supported_act_dtypes(cls) -> List[torch.dtype]:
29
- return [torch.float16]
29
+ return [torch.float16, torch.bfloat16]
30
31
# Need to figure it out
32
def get_min_capability(self) -> int:
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16.py
+12
-9
@@ -64,10 +64,9 @@ class CompressedTensorsW4A16(CompressedTensorsScheme):
64
"input_dim": 1,
65
"output_dim": 0,
66
"packed_dim": 1,
67
- "pack_factor": pack_factor
67
+ "pack_factor": pack_factor,
68
+ "weight_loader": weight_loader
69
})
69
- set_weight_attrs(weight, {"weight_loader": weight_loader})
70
-
70
layer.register_parameter("weight_packed", weight)
71
72
weight_scale = Parameter(
@@ -79,11 +78,12 @@ class CompressedTensorsW4A16(CompressedTensorsScheme):
78
requires_grad=False,
79
)
80
82
- set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
83
- set_weight_attrs(weight_scale, {
84
- "input_dim": weight_scale_dim,
85
- "output_dim": 0
86
- })
81
+ set_weight_attrs(
82
+ weight_scale, {
83
+ "weight_loader": weight_loader,
84
+ "input_dim": weight_scale_dim,
85
+ "output_dim": 0
86
+ })
87
layer.register_parameter("weight_scale", weight_scale)
88
89
# A 2D array defining the original shape of the weights
@@ -92,7 +92,10 @@ class CompressedTensorsW4A16(CompressedTensorsScheme):
92
requires_grad=False)
93
94
layer.register_parameter("weight_shape", weight_shape)
95
- set_weight_attrs(weight_shape, {"weight_loader": weight_loader})
95
+ set_weight_attrs(weight_shape, {
96
+ "weight_loader": weight_loader,
97
+ "ignore_warning": True,
98
+ })
99
100
layer.input_size_per_partition = input_size_per_partition
101
layer.output_size_per_partition = output_size_per_partition
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py
+8
-10
@@ -48,9 +48,6 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
48
weight_scale_dim = sum(
49
output_partition_sizes) if is_tensor_partitioned else 1
50
51
- weight_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
52
- requires_grad=False)
53
-
51
weight_scale = Parameter(torch.empty(weight_scale_dim,
52
dtype=torch.float32),
53
requires_grad=False)
@@ -61,21 +58,22 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
58
requires_grad=False)
59
60
layer.register_parameter("weight", weight)
64
- set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
65
- set_weight_attrs(weight, {"weight_loader": weight_loader})
66
- set_weight_attrs(weight, {"logical_widths": output_partition_sizes})
61
+ set_weight_attrs(
62
+ weight, {
63
+ "input_dim": 1,
64
+ "output_dim": 0,
65
+ "weight_loader": weight_loader,
66
+ "logical_widths": output_partition_sizes
67
+ })
68
69
layer.register_parameter("weight_scale", weight_scale)
69
- set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
70
set_weight_attrs(
71
weight_scale, {
72
+ "weight_loader": weight_loader,
73
"shard_splitter": self.scales_shard_splitter,
74
"logical_widths": output_partition_sizes
75
})
76
76
- layer.register_parameter("weight_zero_point", weight_zero_point)
77
- set_weight_attrs(weight_zero_point, {"weight_loader": weight_loader})
78
-
77
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
78
weight = layer.weight
79
weight_scale = layer.weight_scale
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
-16
@@ -39,22 +39,16 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
39
params_dtype: torch.dtype, weight_loader: Callable,
40
**kwargs):
41
42
- # TODO: remove zero_point parameters once the configs given remove them
43
-
42
is_tensor_partitioned = len(output_partition_sizes) != 1
43
weight_scale_dim = sum(
44
output_partition_sizes) if is_tensor_partitioned else 1
45
46
input_scale = Parameter(torch.empty(1, dtype=torch.float32),
47
requires_grad=False)
50
- input_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
51
- requires_grad=False)
48
49
weight_scale = Parameter(torch.empty(weight_scale_dim,
50
dtype=torch.float32),
51
requires_grad=False)
56
- weight_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
57
- requires_grad=False)
52
53
weight = Parameter(torch.empty(sum(output_partition_sizes),
54
input_size_per_partition,
@@ -72,11 +66,6 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
66
"weight_loader": weight_loader,
67
"ignore_warning": True,
68
})
75
- layer.register_parameter("input_zero_point", input_zero_point)
76
- set_weight_attrs(input_zero_point, {
77
- "weight_loader": weight_loader,
78
- "ignore_warning": True,
79
- })
69
layer.register_parameter("weight_scale", weight_scale)
70
set_weight_attrs(
71
weight_scale, {
@@ -85,11 +74,6 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
74
"logical_widths": output_partition_sizes,
75
"ignore_warning": True,
76
})
88
- layer.register_parameter("weight_zero_point", weight_zero_point)
89
- set_weight_attrs(weight_zero_point, {
90
- "weight_loader": weight_loader,
91
- "ignore_warning": True
92
- })
77
78
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
79
weight = layer.weight