A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Refactor linear layer weight loading; introduce `BasevLLMParameter` and `weight_loader_v2` (#5874)
Dipika Sikka committed
Aug 7, 2024 at 12:17 UTC
0f7052bc7e7c3301588705abf7c7fadf3db293a6
11 files changed
+655
-203
tests/quantization/test_compressed_tensors.py
+12
-8
@@ -9,7 +9,7 @@ import torch
9
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10
CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
11
CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
12
- CompressedTensorsWNA16)
12
+ CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
13
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14
QuantizationType)
15
@@ -109,7 +109,7 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
109
110
assert qkv_proj.weight_packed.dtype is torch.int32
111
assert qkv_proj.weight_scale.dtype is torch.float16
112
- assert qkv_proj.weight_packed.pack_factor == pack_factor
112
+ assert qkv_proj.scheme.pack_factor == pack_factor
113
114
output = llm.generate_greedy("Hello my name is", max_tokens=20)
115
assert output
@@ -140,13 +140,17 @@ def test_compressed_tensors_fp8(vllm_runner):
140
qkv_proj = layer.self_attn.qkv_proj
141
142
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
143
- assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8)
144
- assert qkv_proj.weight.dtype is torch.float8_e4m3fn
143
+ assert isinstance(
144
+ qkv_proj.scheme,
145
+ (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8))
146
+
147
assert qkv_proj.input_scale.dtype is torch.float32
146
- assert qkv_proj.weight_scale.dtype is torch.float32
147
- # should be scalars after processing
148
- assert len(qkv_proj.input_scale.shape) == 0
149
- assert len(qkv_proj.weight_scale.shape) == 0
148
+
149
+ if isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8):
150
+ assert len(qkv_proj.input_scale.shape) == 0
151
+ assert qkv_proj.weight.dtype is torch.float8_e4m3fn
152
+ assert qkv_proj.weight_scale.dtype is torch.float32
153
+ assert len(qkv_proj.weight_scale.shape) == 0
154
155
output = llm.generate_greedy("Hello my name is", max_tokens=20)
156
assert output
vllm/model_executor/__init__.py
+4
@@ -1,7 +1,11 @@
1
+from vllm.model_executor.parameter import (BasevLLMParameter,
2
+ PackedvLLMParameter)
3
from vllm.model_executor.sampling_metadata import SamplingMetadata
4
from vllm.model_executor.utils import set_random_seed
5
6
__all__ = [
7
"SamplingMetadata",
8
"set_random_seed",
9
+ "BasevLLMParameter",
10
+ "PackedvLLMParameter",
11
]
vllm/model_executor/layers/linear.py
+151
-2
@@ -13,10 +13,14 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
13
from vllm.logger import init_logger
14
from vllm.model_executor.layers.quantization.base_config import (
15
QuantizationConfig, QuantizeMethodBase)
16
+from vllm.model_executor.parameter import (BasevLLMParameter,
17
+ PackedvLLMParameter)
18
from vllm.model_executor.utils import set_weight_attrs
19
20
logger = init_logger(__name__)
21
22
+WEIGHT_LOADER_V2_SUPPORTED = ["CompressedTensorsLinearMethod"]
23
+
24
25
def adjust_marlin_shard(param, shard_size, shard_offset):
26
marlin_tile_size = getattr(param, "marlin_tile_size", None)
@@ -288,6 +292,7 @@ class ColumnParallelLinear(LinearBase):
292
293
if output_sizes is None:
294
output_sizes = [output_size]
295
+
296
self.quant_method.create_weights(
297
layer=self,
298
input_size_per_partition=self.input_size,
@@ -295,7 +300,9 @@ class ColumnParallelLinear(LinearBase):
300
input_size=self.input_size,
301
output_size=self.output_size,
302
params_dtype=self.params_dtype,
298
- weight_loader=self.weight_loader,
303
+ weight_loader=(
304
+ self.weight_loader_v2 if self.quant_method.__class__.__name__
305
+ in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader),
306
prefix=prefix)
307
if bias:
308
self.bias = Parameter(
@@ -337,6 +344,9 @@ class ColumnParallelLinear(LinearBase):
344
assert param_data.shape == loaded_weight.shape
345
param_data.copy_(loaded_weight)
346
347
+ def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor):
348
+ param.load_column_parallel_weight(loaded_weight=loaded_weight)
349
+
350
def forward(self, input_):
351
bias = self.bias if not self.skip_bias_add else None
352
@@ -527,6 +537,62 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
537
assert param_data.shape == loaded_weight.shape
538
param_data.copy_(loaded_weight)
539
540
+ def _load_fused_module_from_checkpoint(self, param: BasevLLMParameter,
541
+ loaded_weight: torch.Tensor):
542
+ """
543
+ Handle special case for models where MLP layers are already
544
+ fused on disk. In this case, we have no shard id. This function
545
+ determmines the shard id by splitting these layers and then calls
546
+ the weight loader using the shard id.
547
+
548
+ An example of a model with these fused layers:
549
+ https://huggingface.co/microsoft/Phi-3-mini-4k-instruct
550
+ """
551
+
552
+ current_shard_offset = 0
553
+ shard_offsets: List[Tuple[int, int, int]] = []
554
+ for i, output_size in enumerate(self.output_sizes):
555
+ shard_offsets.append((i, current_shard_offset, output_size))
556
+ current_shard_offset += output_size
557
+
558
+ for shard_id, shard_offset, shard_size in shard_offsets:
559
+ # Special case for Quantization.
560
+ # If quantized, we need to adjust the offset and size to account
561
+ # for the packing.
562
+ if isinstance(param, PackedvLLMParameter
563
+ ) and param.packed_dim == param.output_dim:
564
+ param.adjust_shard_indexes_for_packing(
565
+ shard_size=shard_size, shard_offset=shard_offset)
566
+
567
+ loaded_weight_shard = loaded_weight.narrow(param.output_dim,
568
+ shard_offset,
569
+ shard_size)
570
+ self.weight_loader_v2(param, loaded_weight_shard, shard_id)
571
+
572
+ def weight_loader_v2(self,
573
+ param: BasevLLMParameter,
574
+ loaded_weight: torch.Tensor,
575
+ loaded_shard_id: Optional[int] = None):
576
+ param_data = param.data
577
+ if loaded_shard_id is None:
578
+ if param.output_dim is None:
579
+ assert param_data.shape == loaded_weight.shape
580
+ param_data.copy_(loaded_weight)
581
+ return
582
+ self._load_fused_module_from_checkpoint(param, loaded_weight)
583
+ return
584
+
585
+ assert loaded_shard_id < len(self.output_sizes)
586
+
587
+ tp_size = get_tensor_model_parallel_world_size()
588
+ shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
589
+ shard_size = self.output_sizes[loaded_shard_id] // tp_size
590
+
591
+ param.load_merged_column_weight(loaded_weight=loaded_weight,
592
+ shard_id=loaded_shard_id,
593
+ shard_offset=shard_offset,
594
+ shard_size=shard_size)
595
+
596
597
class QKVParallelLinear(ColumnParallelLinear):
598
"""Linear layers for the attention's QKV transformation.
@@ -598,6 +664,82 @@ class QKVParallelLinear(ColumnParallelLinear):
664
quant_config=quant_config,
665
prefix=prefix)
666
667
+ def _get_shard_offset_mapping(self, loaded_shard_id: str):
668
+ shard_offset_mapping = {
669
+ "q": 0,
670
+ "k": self.num_heads * self.head_size,
671
+ "v": (self.num_heads + self.num_kv_heads) * self.head_size,
672
+ "total": (self.num_heads + 2 * self.num_kv_heads) * self.head_size
673
+ }
674
+ return shard_offset_mapping.get(loaded_shard_id)
675
+
676
+ def _get_shard_size_mapping(self, loaded_shard_id: str):
677
+ shard_size_mapping = {
678
+ "q": self.num_heads * self.head_size,
679
+ "k": self.num_kv_heads * self.head_size,
680
+ "v": self.num_kv_heads * self.head_size,
681
+ }
682
+ return shard_size_mapping.get(loaded_shard_id)
683
+
684
+ def _load_fused_module_from_checkpoint(self, param: BasevLLMParameter,
685
+ loaded_weight: torch.Tensor):
686
+ """
687
+ Handle special case for models where QKV layers are already
688
+ fused on disk. In this case, we have no shard id. This function
689
+ determmines the shard id by splitting these layers and then calls
690
+ the weight loader using the shard id.
691
+
692
+ An example of a model with these fused layers:
693
+ https://huggingface.co/microsoft/Phi-3-mini-4k-instruct
694
+ """
695
+ shard_offsets = [
696
+ # (shard_id, shard_offset, shard_size)
697
+ ("q", 0, self.total_num_heads * self.head_size),
698
+ ("k", self.total_num_heads * self.head_size,
699
+ self.total_num_kv_heads * self.head_size),
700
+ ("v",
701
+ (self.total_num_heads + self.total_num_kv_heads) * self.head_size,
702
+ self.total_num_kv_heads * self.head_size),
703
+ ]
704
+
705
+ for shard_id, shard_offset, shard_size in shard_offsets:
706
+ # Special case for Quantization.
707
+ # If quantized, we need to adjust the offset and size to account
708
+ # for the packing.
709
+ if isinstance(param, PackedvLLMParameter
710
+ ) and param.packed_dim == param.output_dim:
711
+ param.adjust_shard_indexes_for_packing(
712
+ shard_size=shard_size, shard_offset=shard_offset)
713
+
714
+ loaded_weight_shard = loaded_weight.narrow(param.output_dim,
715
+ shard_offset,
716
+ shard_size)
717
+ self.weight_loader_v2(param, loaded_weight_shard, shard_id)
718
+
719
+ def weight_loader_v2(self,
720
+ param: BasevLLMParameter,
721
+ loaded_weight: torch.Tensor,
722
+ loaded_shard_id: Optional[str] = None):
723
+ param_data = param.data
724
+ if loaded_shard_id is None: # special case for certain models
725
+ if param.output_dim is None:
726
+ assert param_data.shape == loaded_weight.shape
727
+ param_data.copy_(loaded_weight)
728
+ return
729
+ self._load_fused_module_from_checkpoint(param, loaded_weight)
730
+ return
731
+
732
+ assert loaded_shard_id in ["q", "k", "v"]
733
+
734
+ shard_offset = self._get_shard_offset_mapping(loaded_shard_id)
735
+ shard_size = self._get_shard_size_mapping(loaded_shard_id)
736
+
737
+ param.load_qkv_weight(loaded_weight=loaded_weight,
738
+ num_heads=self.num_kv_head_replicas,
739
+ shard_id=loaded_shard_id,
740
+ shard_offset=shard_offset,
741
+ shard_size=shard_size)
742
+
743
def weight_loader(self,
744
param: Parameter,
745
loaded_weight: torch.Tensor,
@@ -798,6 +940,7 @@ class RowParallelLinear(LinearBase):
940
self.tp_size = get_tensor_model_parallel_world_size()
941
self.input_size_per_partition = divide(input_size, self.tp_size)
942
assert self.quant_method is not None
943
+
944
self.quant_method.create_weights(
945
layer=self,
946
input_size_per_partition=self.input_size_per_partition,
@@ -805,7 +948,9 @@ class RowParallelLinear(LinearBase):
948
input_size=self.input_size,
949
output_size=self.output_size,
950
params_dtype=self.params_dtype,
808
- weight_loader=self.weight_loader,
951
+ weight_loader=(
952
+ self.weight_loader_v2 if self.quant_method.__class__.__name__
953
+ in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader),
954
prefix=prefix)
955
if not reduce_results and (bias and not skip_bias_add):
956
raise ValueError("When not reduce the results, adding bias to the "
@@ -850,6 +995,10 @@ class RowParallelLinear(LinearBase):
995
assert param_data.shape == loaded_weight.shape
996
param_data.copy_(loaded_weight)
997
998
+ def weight_loader_v2(self, param: BasevLLMParameter,
999
+ loaded_weight: torch.Tensor):
1000
+ param.load_row_parallel_weight(loaded_weight=loaded_weight)
1001
+
1002
def forward(self, input_):
1003
if self.input_is_parallel:
1004
input_parallel = input_
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+9
-12
@@ -19,6 +19,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
19
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
20
from vllm.platforms import current_platform
21
22
+__all__ = ["CompressedTensorsLinearMethod"]
23
+
24
25
class CompressedTensorsConfig(QuantizationConfig):
26
@@ -146,18 +148,15 @@ class CompressedTensorsConfig(QuantizationConfig):
148
if weight_quant is None or input_quant is None:
149
return False
150
149
- # Confirm we have floating points.
150
- if not (weight_quant.type == QuantizationType.FLOAT
151
- and input_quant.type == QuantizationType.FLOAT):
152
- return False
153
-
151
# Confirm weight scheme is supported.
152
+ is_floating_point = (weight_quant.type == QuantizationType.FLOAT
153
+ and input_quant.type == QuantizationType.FLOAT)
154
is_symmetric_weight = weight_quant.symmetric
155
is_static_weight = not weight_quant.dynamic
156
is_per_tensor_or_channel_weight = (weight_quant.strategy in [
157
QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL
158
])
160
- if not (is_symmetric_weight and is_static_weight
159
+ if not (is_floating_point and is_symmetric_weight and is_static_weight
160
and is_per_tensor_or_channel_weight):
161
return False
162
@@ -169,11 +168,7 @@ class CompressedTensorsConfig(QuantizationConfig):
168
is_symmetric_activation = input_quant.symmetric
169
is_per_tensor_activation = (
170
input_quant.strategy == QuantizationStrategy.TENSOR)
172
- if not (is_symmetric_activation and is_per_tensor_activation):
173
- return False
174
-
175
- # All conditions satisfied.
176
- return True
171
+ return is_symmetric_activation and is_per_tensor_activation
172
173
def _is_fp8_w8a16(self, weight_quant: BaseModel,
174
input_quant: BaseModel) -> bool:
@@ -230,6 +225,7 @@ class CompressedTensorsConfig(QuantizationConfig):
225
group_size=weight_quant.group_size)
226
227
# Detect If Activation Quantization.
228
+ # TODO @dsikka: clean-up conditions
229
if is_activation_quantization_format(self.quant_format):
230
if self._is_fp8_w8a8(weight_quant, input_quant):
231
is_fp8_w8a8_supported = self._check_scheme_supported(
@@ -237,7 +233,8 @@ class CompressedTensorsConfig(QuantizationConfig):
233
if is_fp8_w8a8_supported:
234
return CompressedTensorsW8A8Fp8(
235
strategy=weight_quant.strategy,
240
- is_static_input_scheme=(not input_quant.dynamic))
236
+ is_static_input_scheme=(input_quant
237
+ and not input_quant.dynamic))
238
else:
239
return CompressedTensorsW8A16Fp8(
240
strategy=weight_quant.strategy,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
+11
-9
@@ -2,11 +2,10 @@ from typing import Callable, List, Optional
2
3
import torch
4
import torch.nn.functional as F
5
-from torch.nn import Parameter
5
6
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
CompressedTensorsScheme)
9
-from vllm.model_executor.utils import set_weight_attrs
8
+from vllm.model_executor.parameter import ModelWeightParameter
9
10
__all__ = ["CompressedTensorsUnquantized"]
11
@@ -24,7 +23,9 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
23
return 70
24
25
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
27
- pass
26
+ # required by torch.compile to be torch.nn.Parameter
27
+ layer.weight = torch.nn.Parameter(layer.weight.data,
28
+ requires_grad=False)
29
30
def create_weights(self, layer: torch.nn.Module,
31
output_partition_sizes: List[int],
@@ -32,14 +33,15 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
33
params_dtype: torch.dtype, weight_loader: Callable,
34
**kwargs):
35
35
- weight = Parameter(torch.empty(sum(output_partition_sizes),
36
- input_size_per_partition,
37
- dtype=params_dtype),
38
- requires_grad=False)
36
+ weight = ModelWeightParameter(data=torch.empty(
37
+ sum(output_partition_sizes),
38
+ input_size_per_partition,
39
+ dtype=params_dtype),
40
+ input_dim=1,
41
+ output_dim=0,
42
+ weight_loader=weight_loader)
43
40
- set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
44
layer.register_parameter("weight", weight)
42
- set_weight_attrs(weight, {"weight_loader": weight_loader})
45
46
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
47
bias: Optional[torch.Tensor]) -> torch.Tensor:
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+53
-59
@@ -8,7 +8,10 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
9
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
10
GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N)
11
-from vllm.model_executor.utils import set_weight_attrs
11
+from vllm.model_executor.parameter import (BasevLLMParameter,
12
+ ChannelQuantScaleParameter,
13
+ GroupQuantScaleParameter,
14
+ PackedvLLMParameter)
15
from vllm.scalar_type import scalar_types
16
17
__all__ = ["CompressedTensorsW4A16Sparse24"]
@@ -45,7 +48,12 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
48
return 80
49
50
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
48
- pass
51
+ # required by torch.compile to be torch.nn.Parameter
52
+ layer.weight_packed = Parameter(layer.weight_packed.data,
53
+ requires_grad=False)
54
+ layer.scale_packed = Parameter(layer.scale_packed.data,
55
+ requires_grad=False)
56
+ layer.meta = Parameter(layer.meta.data, requires_grad=False)
57
58
def create_weights(self, layer: torch.nn.Module, input_size: int,
59
output_partition_sizes: List[int],
@@ -56,79 +64,65 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
64
pack_factor = 32 // self.quant_type.size_bits
65
output_size_per_partition = sum(output_partition_sizes)
66
59
- qweight = Parameter(
60
- torch.empty(
61
- input_size_per_partition // self.tile_size // 2,
62
- output_size_per_partition * self.tile_size // pack_factor,
63
- dtype=torch.int32,
64
- ),
65
- requires_grad=False,
66
- )
67
- set_weight_attrs(
68
- qweight,
69
- {
70
- "input_dim": 0,
71
- "output_dim": 1,
72
- "packed_dim": 1,
73
- "pack_factor": pack_factor,
74
- "marlin_tile_size": self.tile_size,
75
- "weight_loader": weight_loader
76
- },
77
- )
78
-
79
- layer.register_parameter("weight_packed", qweight)
67
+ qweight = PackedvLLMParameter(data=torch.empty(
68
+ input_size_per_partition // self.tile_size // 2,
69
+ output_size_per_partition * self.tile_size // pack_factor,
70
+ dtype=torch.int32,
71
+ ),
72
+ input_dim=0,
73
+ output_dim=1,
74
+ packed_dim=1,
75
+ packed_factor=pack_factor,
76
+ marlin_tile_size=self.tile_size,
77
+ weight_loader=weight_loader)
78
79
input_groups = (1 if self.group_size is None else
80
input_size_per_partition // self.group_size)
81
84
- scales = Parameter(
82
+ weight_scale_args = {
83
+ "data":
84
torch.empty(
85
input_groups,
86
output_size_per_partition,
87
dtype=params_dtype,
88
),
90
- requires_grad=False,
91
- )
92
- set_weight_attrs(
93
- scales,
94
- {
95
- "output_dim": 1,
96
- "input_dim": None if input_groups == 1 else 0,
97
- "weight_loader": weight_loader
98
- },
99
- )
100
- layer.register_parameter("scale_packed", scales)
101
-
102
- weight_shape = Parameter(torch.empty(2, dtype=torch.int64),
103
- requires_grad=False)
89
+ "weight_loader":
90
+ weight_loader
91
+ }
92
+
93
+ if self.group_size is not None:
94
+ scales = GroupQuantScaleParameter(output_dim=1,
95
+ input_dim=0,
96
+ **weight_scale_args)
97
+ else:
98
+ scales = ChannelQuantScaleParameter(output_dim=1,
99
+ **weight_scale_args)
100
+
101
+ weight_shape = BasevLLMParameter(data=torch.empty(2,
102
+ dtype=torch.int64),
103
+ weight_loader=weight_loader)
104
+
105
+ meta = PackedvLLMParameter(data=torch.empty(
106
+ input_size_per_partition // 8 // 2 // 2,
107
+ output_size_per_partition * 2,
108
+ dtype=torch.int16,
109
+ ),
110
+ input_dim=0,
111
+ output_dim=1,
112
+ packed_dim=1,
113
+ packed_factor=1,
114
+ marlin_tile_size=2,
115
+ weight_loader=weight_loader)
116
117
+ layer.register_parameter("weight_packed", qweight)
118
layer.register_parameter("weight_shape", weight_shape)
106
- set_weight_attrs(weight_shape, {"weight_loader": weight_loader})
107
-
108
- meta = Parameter(
109
- torch.empty(
110
- input_size_per_partition // 8 // 2 // 2,
111
- output_size_per_partition * 2,
112
- dtype=torch.int16,
113
- ),
114
- requires_grad=False,
115
- )
116
- set_weight_attrs(
117
- meta,
118
- {
119
- "input_dim": 0,
120
- "packed_dim": 1,
121
- "pack_factor": 1,
122
- "output_dim": 1,
123
- "marlin_tile_size": 2,
124
- "weight_loader": weight_loader
125
- },
126
- )
119
+ layer.register_parameter("scale_packed", scales)
120
layer.register_parameter("meta", meta)
121
122
max_workspace_size = (
123
output_size_per_partition //
124
GPTQ_MARLIN_24_MIN_THREAD_N) * GPTQ_MARLIN_24_MAX_PARALLEL
125
+
126
workspace = Parameter(torch.zeros(max_workspace_size, dtype=torch.int),
127
requires_grad=False)
128
layer.workspace = workspace
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+32
-19
@@ -9,9 +9,10 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
10
apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
11
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12
- convert_to_channelwise, create_per_channel_scale_param,
13
- create_per_tensor_scale_param)
14
-from vllm.model_executor.utils import set_weight_attrs
12
+ convert_to_channelwise)
13
+from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
14
+ ModelWeightParameter,
15
+ PerTensorScaleParameter)
16
17
__all__ = ["CompressedTensorsW8A16Fp8"]
18
@@ -40,11 +41,19 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
41
layer.logical_widths)
42
layer.weight_scale = torch.nn.Parameter(ws_channelwise,
43
requires_grad=False)
44
+ else:
45
+ # required by torch.compile to be torch.nn.Parameter
46
+ layer.weight_scale = torch.nn.Parameter(layer.weight_scale.data,
47
+ requires_grad=False)
48
49
# Weights must be transposed for marlin
50
layer.weight = torch.nn.Parameter(layer.weight.t(),
51
requires_grad=False)
52
53
+ if self.is_static_input_scheme:
54
+ # required by torch.compile to be torch.nn.Parameter
55
+ layer.input_scale = torch.nn.Parameter(layer.input_scale.data,
56
+ requires_grad=False)
57
prepare_fp8_layer_for_marlin(layer, strategy="channel")
58
59
def create_weights(self, layer: torch.nn.Module, input_size: int,
@@ -60,35 +69,39 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
69
layer.orig_dtype = params_dtype
70
71
# WEIGHT
63
- weight = torch.nn.Parameter(torch.empty(output_size_per_partition,
64
- input_size_per_partition,
65
- dtype=torch.float8_e4m3fn),
66
- requires_grad=False)
72
+ weight = ModelWeightParameter(data=torch.empty(
73
+ output_size_per_partition,
74
+ input_size_per_partition,
75
+ dtype=torch.float8_e4m3fn),
76
+ input_dim=1,
77
+ output_dim=0,
78
+ weight_loader=weight_loader)
79
layer.register_parameter("weight", weight)
68
- set_weight_attrs(weight, {
69
- "input_dim": 1,
70
- "output_dim": 0,
71
- "weight_loader": weight_loader,
72
- })
80
81
# WEIGHT SCALE
75
- layer_kwargs = {"weight_loader": weight_loader}
82
if self.strategy == QuantizationStrategy.CHANNEL:
77
- weight_scale = create_per_channel_scale_param(
78
- output_partition_sizes, **layer_kwargs)
83
+ weight_scale = ChannelQuantScaleParameter(
84
+ data=torch.empty((sum(output_partition_sizes), 1),
85
+ dtype=torch.float32),
86
+ output_dim=0,
87
+ weight_loader=weight_loader)
88
elif self.strategy == QuantizationStrategy.TENSOR:
80
- weight_scale = create_per_tensor_scale_param(
81
- output_partition_sizes, **layer_kwargs)
89
+ weight_scale = PerTensorScaleParameter(data=torch.empty(
90
+ len(output_partition_sizes), dtype=torch.float32),
91
+ weight_loader=weight_loader)
92
else:
93
raise ValueError(
94
f"Unsupported weight strategy={self.strategy}, "
95
f"supported strategies are {SUPPORTED_STRATEGIES}")
96
+
97
+ weight_scale[:] = torch.finfo(torch.float32).min
98
layer.register_parameter("weight_scale", weight_scale)
99
100
# INPUT SCALE (to deal with converted checkpoints)
101
if self.is_static_input_scheme:
90
- input_scale = create_per_tensor_scale_param(
91
- output_partition_sizes, **layer_kwargs)
102
+ input_scale = PerTensorScaleParameter(data=torch.empty(
103
+ len(output_partition_sizes), dtype=torch.float32),
104
+ weight_loader=weight_loader)
105
layer.register_parameter("input_scale", input_scale)
106
107
def apply_weights(self,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+31
-20
@@ -8,10 +8,10 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
QuantizationStrategy)
10
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11
- apply_fp8_linear, create_per_channel_scale_param,
12
- create_per_tensor_scale_param, cutlass_fp8_supported,
13
- requantize_with_max_scale)
14
-from vllm.model_executor.utils import set_weight_attrs
11
+ apply_fp8_linear, cutlass_fp8_supported, requantize_with_max_scale)
12
+from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
13
+ ModelWeightParameter,
14
+ PerTensorScaleParameter)
15
16
__all__ = ["CompressedTensorsW8A8Fp8"]
17
@@ -46,6 +46,9 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
46
elif self.strategy == QuantizationStrategy.CHANNEL:
47
weight = layer.weight
48
layer.weight = Parameter(weight.t(), requires_grad=False)
49
+ # required by torch.compile to be torch.nn.Parameter
50
+ layer.weight_scale = Parameter(layer.weight_scale.data,
51
+ requires_grad=False)
52
53
else:
54
raise ValueError(f"Unknown quantization strategy {self.strategy}")
@@ -66,32 +69,40 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
69
layer.logical_widths = output_partition_sizes
70
71
# WEIGHT
69
- weight = torch.nn.Parameter(torch.empty(output_size_per_partition,
70
- input_size_per_partition,
71
- dtype=torch.float8_e4m3fn),
72
- requires_grad=False)
72
+ weight = ModelWeightParameter(data=torch.empty(
73
+ output_size_per_partition,
74
+ input_size_per_partition,
75
+ dtype=torch.float8_e4m3fn),
76
+ input_dim=1,
77
+ output_dim=0,
78
+ weight_loader=weight_loader)
79
layer.register_parameter("weight", weight)
74
- set_weight_attrs(weight, {
75
- "input_dim": 1,
76
- "output_dim": 0,
77
- "weight_loader": weight_loader,
78
- })
80
81
# WEIGHT SCALE
81
- layer_kwargs = {"weight_loader": weight_loader}
82
+ # TODO: update create_xxx_parameter functions to return
83
+ # the newly added parameters
84
if self.strategy == QuantizationStrategy.CHANNEL:
83
- weight_scale = create_per_channel_scale_param(
84
- output_partition_sizes, **layer_kwargs)
85
+ weight_scale = ChannelQuantScaleParameter(
86
+ data=torch.empty((sum(output_partition_sizes), 1),
87
+ dtype=torch.float32),
88
+ output_dim=0,
89
+ weight_loader=weight_loader)
90
else:
91
assert self.strategy == QuantizationStrategy.TENSOR
87
- weight_scale = create_per_tensor_scale_param(
88
- output_partition_sizes, **layer_kwargs)
92
+ weight_scale = PerTensorScaleParameter(data=torch.empty(
93
+ len(output_partition_sizes), dtype=torch.float32),
94
+ weight_loader=weight_loader)
95
+
96
+ # min requirement for fp8 kernels
97
+ weight_scale[:] = torch.finfo(torch.float32).min
98
layer.register_parameter("weight_scale", weight_scale)
99
100
# INPUT SCALE
101
if self.is_static_input_scheme:
93
- input_scale = create_per_tensor_scale_param(
94
- output_partition_sizes, **layer_kwargs)
102
+ input_scale = PerTensorScaleParameter(data=torch.empty(
103
+ len(output_partition_sizes), dtype=torch.float32),
104
+ weight_loader=weight_loader)
105
+ input_scale[:] = torch.finfo(torch.float32).min
106
layer.register_parameter("input_scale", input_scale)
107
108
def apply_weights(self,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+27
-20
@@ -8,9 +8,11 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
QuantizationStrategy)
10
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11
- apply_int8_linear, convert_to_channelwise, create_per_channel_scale_param,
12
- create_per_tensor_scale_param)
13
-from vllm.model_executor.utils import set_weight_attrs
11
+ apply_int8_linear, convert_to_channelwise)
12
+from vllm.model_executor.parameter import (BasevLLMParameter,
13
+ ChannelQuantScaleParameter,
14
+ ModelWeightParameter,
15
+ PerTensorScaleParameter)
16
17
18
class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
@@ -39,7 +41,9 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
41
ws_channelwise = convert_to_channelwise(layer.weight_scale,
42
self.logical_widths)
43
layer.weight_scale = Parameter(ws_channelwise, requires_grad=False)
42
-
44
+ else:
45
+ layer.weight_scale = Parameter(layer.weight_scale.data,
46
+ requires_grad=False)
47
# INPUT SCALE
48
if self.is_static_input_scheme:
49
layer.input_scale = Parameter(layer.input_scale.max(),
@@ -55,32 +59,35 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
59
self.logical_widths = output_partition_sizes
60
61
# WEIGHT
58
- weight = Parameter(torch.empty(sum(output_partition_sizes),
59
- input_size_per_partition,
60
- dtype=torch.int8),
61
- requires_grad=False)
62
+ weight = ModelWeightParameter(data=torch.empty(
63
+ sum(output_partition_sizes),
64
+ input_size_per_partition,
65
+ dtype=torch.int8),
66
+ input_dim=1,
67
+ output_dim=0,
68
+ weight_loader=weight_loader)
69
+
70
layer.register_parameter("weight", weight)
63
- set_weight_attrs(weight, {
64
- "input_dim": 1,
65
- "output_dim": 0,
66
- "weight_loader": weight_loader,
67
- })
71
72
# WEIGHT SCALE
70
- layer_kwargs = {"weight_loader": weight_loader}
73
if self.strategy == QuantizationStrategy.CHANNEL:
72
- weight_scale = create_per_channel_scale_param(
73
- output_partition_sizes, **layer_kwargs)
74
+ weight_scale = ChannelQuantScaleParameter(
75
+ data=torch.empty((sum(output_partition_sizes), 1),
76
+ dtype=torch.float32),
77
+ output_dim=0,
78
+ weight_loader=weight_loader)
79
else:
80
assert self.strategy == QuantizationStrategy.TENSOR
76
- weight_scale = create_per_tensor_scale_param(
77
- output_partition_sizes, **layer_kwargs)
81
+ weight_scale = PerTensorScaleParameter(data=torch.empty(
82
+ len(output_partition_sizes), dtype=torch.float32),
83
+ weight_loader=weight_loader)
84
layer.register_parameter("weight_scale", weight_scale)
85
86
# INPUT SCALE
87
if self.is_static_input_scheme:
82
- input_scale = create_per_tensor_scale_param(
83
- output_partition_sizes, **layer_kwargs)
88
+ input_scale = BasevLLMParameter(data=torch.empty(
89
+ 1, dtype=torch.float32),
90
+ weight_loader=weight_loader)
91
layer.register_parameter("input_scale", input_scale)
92
93
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+48
-54
@@ -1,7 +1,6 @@
1
from typing import Callable, List, Optional
2
3
import torch
4
-from torch.nn import Parameter
4
5
from vllm import _custom_ops as ops
6
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
@@ -10,7 +9,10 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
9
apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
10
marlin_permute_scales, replace_tensor, verify_marlin_supported,
11
verify_marlin_supports_shape)
13
-from vllm.model_executor.utils import set_weight_attrs
12
+from vllm.model_executor.parameter import (BasevLLMParameter,
13
+ ChannelQuantScaleParameter,
14
+ GroupQuantScaleParameter,
15
+ PackedvLLMParameter)
16
from vllm.scalar_type import scalar_types
17
18
__all__ = ["CompressedTensorsWNA16"]
@@ -30,17 +32,12 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
32
33
self.pack_factor = 32 // num_bits
34
self.strategy = strategy
35
+ self.group_size = -1 if group_size is None else group_size
36
34
- self.group_size: int
35
- if group_size is None:
36
- if self.strategy != "channel":
37
- raise ValueError(
38
- "Marlin kernels require group quantization or "
39
- "channelwise quantization, but found no group "
40
- "size and strategy is not channelwise.")
41
- self.group_size = -1
42
- else:
43
- self.group_size = group_size
37
+ if self.group_size == -1 and self.strategy != "channel":
38
+ raise ValueError("Marlin kernels require group quantization or "
39
+ "channelwise quantization, but found no group "
40
+ "size and strategy is not channelwise.")
41
42
if num_bits not in WNA16_SUPPORTED_TYPES_MAP:
43
raise ValueError(
@@ -63,11 +60,12 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
60
input_size_per_partition: int,
61
params_dtype: torch.dtype, weight_loader: Callable,
62
**kwargs):
63
+
64
output_size_per_partition = sum(output_partition_sizes)
65
66
# If group_size is -1, we are in channelwise case.
67
channelwise = (self.group_size == -1)
70
- group_size = input_size if channelwise else self.group_size
68
+ group_size = self.group_size if self.group_size != -1 else input_size
69
row_parallel = (input_size != input_size_per_partition)
70
# In the case of channelwise quantization, we need to replicate the
71
# scales across all gpus.
@@ -79,60 +77,51 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
77
input_size=input_size,
78
group_size=group_size)
79
82
- weight_scale_dim = None
80
scales_and_zp_size = input_size // group_size
81
82
if partition_scales:
83
assert input_size_per_partition % group_size == 0
87
- weight_scale_dim = 1
84
scales_and_zp_size = input_size_per_partition // group_size
85
90
- weight = Parameter(
91
- torch.empty(
92
- output_size_per_partition,
93
- input_size_per_partition // self.pack_factor,
94
- dtype=torch.int32,
95
- ),
96
- requires_grad=False,
97
- )
98
-
99
- set_weight_attrs(
100
- weight, {
101
- "input_dim": 1,
102
- "output_dim": 0,
103
- "packed_dim": 1,
104
- "pack_factor": self.pack_factor,
105
- "weight_loader": weight_loader
106
- })
107
- layer.register_parameter("weight_packed", weight)
108
-
109
- weight_scale = Parameter(
86
+ weight = PackedvLLMParameter(input_dim=1,
87
+ output_dim=0,
88
+ weight_loader=weight_loader,
89
+ packed_factor=self.pack_factor,
90
+ packed_dim=1,
91
+ data=torch.empty(
92
+ output_size_per_partition,
93
+ input_size_per_partition //
94
+ self.pack_factor,
95
+ dtype=torch.int32,
96
+ ))
97
+
98
+ weight_scale_args = {
99
+ "weight_loader":
100
+ weight_loader,
101
+ "data":
102
torch.empty(
103
output_size_per_partition,
104
scales_and_zp_size,
105
dtype=params_dtype,
114
- ),
115
- requires_grad=False,
116
- )
117
-
118
- set_weight_attrs(
119
- weight_scale, {
120
- "weight_loader": weight_loader,
121
- "input_dim": weight_scale_dim,
122
- "output_dim": 0
123
- })
124
- layer.register_parameter("weight_scale", weight_scale)
106
+ )
107
+ }
108
+ if self.group_size == -1:
109
+ weight_scale = ChannelQuantScaleParameter(output_dim=0,
110
+ **weight_scale_args)
111
+ else:
112
+ weight_scale = GroupQuantScaleParameter(output_dim=0,
113
+ input_dim=1,
114
+ **weight_scale_args)
115
116
# A 2D array defining the original shape of the weights
117
# before packing
128
- weight_shape = Parameter(torch.empty(2, dtype=torch.int64),
129
- requires_grad=False)
118
+ weight_shape = BasevLLMParameter(data=torch.empty(2,
119
+ dtype=torch.int64),
120
+ weight_loader=weight_loader)
121
122
+ layer.register_parameter("weight_packed", weight)
123
+ layer.register_parameter("weight_scale", weight_scale)
124
layer.register_parameter("weight_shape", weight_shape)
132
- set_weight_attrs(weight_shape, {
133
- "weight_loader": weight_loader,
134
- "ignore_warning": True,
135
- })
125
126
layer.input_size_per_partition = input_size_per_partition
127
layer.output_size_per_partition = output_size_per_partition
@@ -154,10 +143,15 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
143
144
# No zero-point
145
layer.weight_zp = marlin_make_empty_g_idx(device)
146
+ # Update for kernel
147
+ layer.weight_packed = torch.nn.Parameter(
148
+ layer.weight_packed.t().contiguous(), requires_grad=False)
149
+ layer.weight_scale = torch.nn.Parameter(
150
+ layer.weight_scale.squeeze().t().contiguous(), requires_grad=False)
151
152
# Repack weights from compressed-tensors format to marlin format.
153
marlin_qweight = ops.gptq_marlin_repack(
160
- layer.weight_packed.t().contiguous(),
154
+ layer.weight_packed,
155
perm=layer.g_idx_sort_indices,
156
size_k=layer.input_size_per_partition,
157
size_n=layer.output_size_per_partition,
@@ -166,7 +160,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
160
161
# Permute scales from compressed-tensors format to marlin format.
162
marlin_scales = marlin_permute_scales(
169
- layer.weight_scale.squeeze().t().contiguous(),
163
+ layer.weight_scale,
164
size_k=layer.input_size_per_partition,
165
size_n=layer.output_size_per_partition,
166
group_size=layer.group_size)
vllm/model_executor/parameter.py
new
+277
@@ -0,0 +1,277 @@
1
+from typing import Callable, Optional, Union
2
+
3
+import torch
4
+from torch.nn import Parameter
5
+
6
+from vllm.distributed import get_tensor_model_parallel_rank
7
+from vllm.logger import init_logger
8
+
9
+__all__ = [
10
+ "BasevLLMParameter", "PackedvLLMParameter", "PerTensorScaleParameter",
11
+ "ModelWeightParameter", "ChannelQuantScaleParameter",
12
+ "GroupQuantScaleParameter"
13
+]
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+class BasevLLMParameter(Parameter):
19
+ """
20
+ Base parameter for vLLM linear layers. Extends the torch.nn.parameter
21
+ by taking in a linear weight loader. Will copy the loaded weight
22
+ into the parameter when the provided weight loader is called.
23
+ """
24
+
25
+ def __new__(cls, data: torch.Tensor, **kwargs):
26
+
27
+ return super().__new__(cls, data=data, requires_grad=False)
28
+
29
+ def __init__(self, data: torch.Tensor, weight_loader: Callable):
30
+ """
31
+ Initialize the BasevLLMParameter
32
+
33
+ :param data: torch tensor with the parameter data
34
+ :param weight_loader: weight loader callable
35
+
36
+ :returns: a torch.nn.parameter
37
+ """
38
+
39
+ self._weight_loader = weight_loader
40
+
41
+ @property
42
+ def weight_loader(self):
43
+ return self._weight_loader
44
+
45
+ def _assert_and_load(self, loaded_weight: torch.Tensor):
46
+ assert self.data.shape == loaded_weight.shape
47
+ self.data.copy_(loaded_weight)
48
+
49
+ def load_column_parallel_weight(self, loaded_weight: torch.Tensor):
50
+ self._assert_and_load(loaded_weight)
51
+
52
+ def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
53
+ self._assert_and_load(loaded_weight)
54
+
55
+ def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
56
+ self._assert_and_load(loaded_weight)
57
+
58
+ def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
59
+ self._assert_and_load(loaded_weight)
60
+
61
+
62
+class _ColumnvLLMParameter(BasevLLMParameter):
63
+ """
64
+ Private class defining weight loading functionality
65
+ (load_merged_column_weight, load_qkv_weight)
66
+ for parameters being loaded into linear layers with column
67
+ parallelism. This includes QKV and MLP layers which are
68
+ not already fused on disk. Requires an output dimension
69
+ to be defined. Called within the weight loader of
70
+ each of the column parallel linear layers.
71
+ """
72
+
73
+ def __init__(self, output_dim: int, **kwargs):
74
+ self._output_dim = output_dim
75
+ super().__init__(**kwargs)
76
+
77
+ @property
78
+ def output_dim(self):
79
+ return self._output_dim
80
+
81
+ def load_column_parallel_weight(self, loaded_weight: torch.Tensor):
82
+ tp_rank = get_tensor_model_parallel_rank()
83
+ shard_size = self.data.shape[self.output_dim]
84
+ loaded_weight = loaded_weight.narrow(self.output_dim,
85
+ tp_rank * shard_size, shard_size)
86
+ assert self.data.shape == loaded_weight.shape
87
+ self.data.copy_(loaded_weight)
88
+
89
+ def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
90
+
91
+ shard_offset = kwargs.get("shard_offset")
92
+ shard_size = kwargs.get("shard_size")
93
+ if isinstance(
94
+ self,
95
+ PackedvLLMParameter) and self.packed_dim == self.output_dim:
96
+ shard_size, shard_offset = self.adjust_shard_indexes_for_packing(
97
+ shard_offset=shard_offset, shard_size=shard_size)
98
+
99
+ param_data = self.data
100
+
101
+ tp_rank = get_tensor_model_parallel_rank()
102
+ param_data = param_data.narrow(self.output_dim, shard_offset,
103
+ shard_size)
104
+ loaded_weight = loaded_weight.narrow(self.output_dim,
105
+ tp_rank * shard_size, shard_size)
106
+ assert param_data.shape == loaded_weight.shape
107
+ param_data.copy_(loaded_weight)
108
+
109
+ def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
110
+
111
+ shard_offset = kwargs.get("shard_offset")
112
+ shard_size = kwargs.get("shard_size")
113
+ shard_id = kwargs.get("shard_id")
114
+ num_heads = kwargs.get("num_heads")
115
+
116
+ if isinstance(
117
+ self,
118
+ PackedvLLMParameter) and self.output_dim == self.packed_dim:
119
+ shard_size, shard_offset = self.adjust_shard_indexes_for_packing(
120
+ shard_offset=shard_offset, shard_size=shard_size)
121
+
122
+ param_data = self.data
123
+ tp_rank = get_tensor_model_parallel_rank()
124
+ shard_id = tp_rank if shard_id == "q" else tp_rank // num_heads
125
+ param_data = param_data.narrow(self.output_dim, shard_offset,
126
+ shard_size)
127
+ loaded_weight = loaded_weight.narrow(self.output_dim,
128
+ shard_id * shard_size, shard_size)
129
+
130
+ assert param_data.shape == loaded_weight.shape
131
+ param_data.copy_(loaded_weight)
132
+
133
+
134
+class ModelWeightParameter(_ColumnvLLMParameter):
135
+ """
136
+ Parameter class for linear layer weights. Extends the
137
+ _ColumnvLLMParameter by adding loading functionality
138
+ for linear layers with row parallel functionality.
139
+ Requires an input dimension to be defined.
140
+ """
141
+
142
+ def __init__(self, input_dim: int, **kwargs):
143
+ self._input_dim = input_dim
144
+ super().__init__(**kwargs)
145
+
146
+ @property
147
+ def input_dim(self):
148
+ return self._input_dim
149
+
150
+ def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
151
+ tp_rank = get_tensor_model_parallel_rank()
152
+ shard_size = self.data.shape[self.input_dim]
153
+ loaded_weight = loaded_weight.narrow(self.input_dim,
154
+ tp_rank * shard_size, shard_size)
155
+
156
+ if len(loaded_weight.shape) == 0:
157
+ loaded_weight = loaded_weight.reshape(1)
158
+
159
+ assert self.data.shape == loaded_weight.shape
160
+ self.data.copy_(loaded_weight)
161
+
162
+
163
+class GroupQuantScaleParameter(ModelWeightParameter):
164
+ """
165
+ Parameter class for weight scales loaded for weights with
166
+ grouped quantization. Equivalent to ModelWeightParameter.
167
+ """
168
+ pass
169
+
170
+
171
+class ChannelQuantScaleParameter(_ColumnvLLMParameter):
172
+ """
173
+ Parameter class for weight scales loaded for weights with
174
+ channel-wise quantization. Equivalent to _ColumnvLLMParameter.
175
+ """
176
+ pass
177
+
178
+
179
+class PerTensorScaleParameter(BasevLLMParameter):
180
+ """
181
+ Parameter class for scales where the number of scales is
182
+ equivalent to the number of logical matrices in fused linear
183
+ layers (e.g. for QKV, there are 3 scales loaded from disk).
184
+ This is relevant to weights with per-tensor quantization.
185
+ Adds functionality to map the scalers to a shard during
186
+ weight loading.
187
+
188
+ Note: additional parameter manipulation may be handled
189
+ for each quantization config specifically, within
190
+ process_weights_after_loading
191
+ """
192
+
193
+ def __init__(self, **kwargs):
194
+ self.qkv_idxs = {"q": 0, "k": 1, "v": 2}
195
+ super().__init__(**kwargs)
196
+
197
+ def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
198
+ if isinstance(shard_id, int):
199
+ return shard_id
200
+
201
+ assert isinstance(shard_id, str)
202
+ assert shard_id in self.qkv_idxs
203
+ return self.qkv_idxs[shard_id]
204
+
205
+ def load_merged_column_weight(self, *args, **kwargs):
206
+ self._load_into_shard_id(*args, **kwargs)
207
+
208
+ def load_qkv_weight(self, *args, **kwargs):
209
+ self._load_into_shard_id(*args, **kwargs)
210
+
211
+ def load_column_parallel_weight(self, *args, **kwargs):
212
+ self._load_into_shard_id(*args, **kwargs)
213
+
214
+ def _load_into_shard_id(self, loaded_weight: torch.Tensor,
215
+ shard_id: Union[str, int], **kwargs):
216
+ """
217
+ Slice the parameter data based on the shard id for
218
+ loading.
219
+ """
220
+
221
+ param_data = self.data
222
+ shard_id = self._shard_id_as_int(shard_id)
223
+
224
+ # AutoFP8 scales do not have a shape
225
+ # compressed-tensors scales do have a shape
226
+ if len(loaded_weight.shape) != 0:
227
+ assert loaded_weight.shape[0] == 1
228
+ loaded_weight = loaded_weight[0]
229
+
230
+ param_data = param_data[shard_id]
231
+ assert param_data.shape == loaded_weight.shape
232
+ param_data.copy_(loaded_weight)
233
+
234
+
235
+class PackedvLLMParameter(ModelWeightParameter):
236
+ """
237
+ Parameter for model weights which are packed on disk.
238
+ Example: GPTQ Marlin weights are int4 or int8, packed into int32.
239
+ Extends the ModelWeightParameter to take in the
240
+ packed factor, the packed dimension, and optionally, marlin
241
+ tile size for marlin kernels. Adjusts the shard_size and
242
+ shard_offset for fused linear layers model weight loading
243
+ by accounting for packing and optionally, marlin tile size.
244
+ """
245
+
246
+ def __init__(self,
247
+ packed_factor: int,
248
+ packed_dim: int,
249
+ marlin_tile_size: Optional[int] = None,
250
+ **kwargs):
251
+ self._packed_factor = packed_factor
252
+ self._packed_dim = packed_dim
253
+ self._marlin_tile = marlin_tile_size
254
+ super().__init__(**kwargs)
255
+
256
+ @property
257
+ def packed_dim(self):
258
+ return self._packed_dim
259
+
260
+ @property
261
+ def packed_factor(self):
262
+ return self._packed_factor
263
+
264
+ @property
265
+ def marlin_tile(self):
266
+ return self._marlin_tile
267
+
268
+ def _adjust_shard_indexes_for_marlin(self, shard_size, shard_offset):
269
+ return shard_size * self.marlin_tile, shard_offset * self.marlin_tile
270
+
271
+ def adjust_shard_indexes_for_packing(self, shard_size, shard_offset):
272
+ shard_size = shard_size // self.packed_factor
273
+ shard_offset = shard_offset // self.packed_factor
274
+ if self.marlin_tile is not None:
275
+ return self._adjust_shard_indexes_for_marlin(
276
+ shard_size, shard_offset)
277
+ return shard_size, shard_offset