@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f7052bc7e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Refactor linear layer weight loading; introduce `BasevLLMParameter` and `weight_loader_v2` (#5874)

Dipika Sikka committed Aug 7, 2024 at 12:17 UTC 0f7052bc7e7c3301588705abf7c7fadf3db293a6
11 files changed +655 -203
tests/quantization/test_compressed_tensors.py
+12 -8
@@ -9,7 +9,7 @@ import torch
9 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10 CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
11 CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
12 - CompressedTensorsWNA16)
12 + CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
13 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
14 QuantizationType)
15
@@ -109,7 +109,7 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
109
110 assert qkv_proj.weight_packed.dtype is torch.int32
111 assert qkv_proj.weight_scale.dtype is torch.float16
112 - assert qkv_proj.weight_packed.pack_factor == pack_factor
112 + assert qkv_proj.scheme.pack_factor == pack_factor
113
114 output = llm.generate_greedy("Hello my name is", max_tokens=20)
115 assert output
@@ -140,13 +140,17 @@ def test_compressed_tensors_fp8(vllm_runner):
140 qkv_proj = layer.self_attn.qkv_proj
141
142 assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
143 - assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8)
144 - assert qkv_proj.weight.dtype is torch.float8_e4m3fn
143 + assert isinstance(
144 + qkv_proj.scheme,
145 + (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8))
146 +
147 assert qkv_proj.input_scale.dtype is torch.float32
146 - assert qkv_proj.weight_scale.dtype is torch.float32
147 - # should be scalars after processing
148 - assert len(qkv_proj.input_scale.shape) == 0
149 - assert len(qkv_proj.weight_scale.shape) == 0
148 +
149 + if isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8):
150 + assert len(qkv_proj.input_scale.shape) == 0
151 + assert qkv_proj.weight.dtype is torch.float8_e4m3fn
152 + assert qkv_proj.weight_scale.dtype is torch.float32
153 + assert len(qkv_proj.weight_scale.shape) == 0
154
155 output = llm.generate_greedy("Hello my name is", max_tokens=20)
156 assert output
vllm/model_executor/__init__.py
+4
@@ -1,7 +1,11 @@
1 +from vllm.model_executor.parameter import (BasevLLMParameter,
2 + PackedvLLMParameter)
3 from vllm.model_executor.sampling_metadata import SamplingMetadata
4 from vllm.model_executor.utils import set_random_seed
5
6 __all__ = [
7 "SamplingMetadata",
8 "set_random_seed",
9 + "BasevLLMParameter",
10 + "PackedvLLMParameter",
11 ]
vllm/model_executor/layers/linear.py
+151 -2
@@ -13,10 +13,14 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
13 from vllm.logger import init_logger
14 from vllm.model_executor.layers.quantization.base_config import (
15 QuantizationConfig, QuantizeMethodBase)
16 +from vllm.model_executor.parameter import (BasevLLMParameter,
17 + PackedvLLMParameter)
18 from vllm.model_executor.utils import set_weight_attrs
19
20 logger = init_logger(__name__)
21
22 +WEIGHT_LOADER_V2_SUPPORTED = ["CompressedTensorsLinearMethod"]
23 +
24
25 def adjust_marlin_shard(param, shard_size, shard_offset):
26 marlin_tile_size = getattr(param, "marlin_tile_size", None)
@@ -288,6 +292,7 @@ class ColumnParallelLinear(LinearBase):
292
293 if output_sizes is None:
294 output_sizes = [output_size]
295 +
296 self.quant_method.create_weights(
297 layer=self,
298 input_size_per_partition=self.input_size,
@@ -295,7 +300,9 @@ class ColumnParallelLinear(LinearBase):
300 input_size=self.input_size,
301 output_size=self.output_size,
302 params_dtype=self.params_dtype,
298 - weight_loader=self.weight_loader,
303 + weight_loader=(
304 + self.weight_loader_v2 if self.quant_method.__class__.__name__
305 + in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader),
306 prefix=prefix)
307 if bias:
308 self.bias = Parameter(
@@ -337,6 +344,9 @@ class ColumnParallelLinear(LinearBase):
344 assert param_data.shape == loaded_weight.shape
345 param_data.copy_(loaded_weight)
346
347 + def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor):
348 + param.load_column_parallel_weight(loaded_weight=loaded_weight)
349 +
350 def forward(self, input_):
351 bias = self.bias if not self.skip_bias_add else None
352
@@ -527,6 +537,62 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
537 assert param_data.shape == loaded_weight.shape
538 param_data.copy_(loaded_weight)
539
540 + def _load_fused_module_from_checkpoint(self, param: BasevLLMParameter,
541 + loaded_weight: torch.Tensor):
542 + """
543 + Handle special case for models where MLP layers are already
544 + fused on disk. In this case, we have no shard id. This function
545 + determmines the shard id by splitting these layers and then calls
546 + the weight loader using the shard id.
547 +
548 + An example of a model with these fused layers:
549 + https://huggingface.co/microsoft/Phi-3-mini-4k-instruct
550 + """
551 +
552 + current_shard_offset = 0
553 + shard_offsets: List[Tuple[int, int, int]] = []
554 + for i, output_size in enumerate(self.output_sizes):
555 + shard_offsets.append((i, current_shard_offset, output_size))
556 + current_shard_offset += output_size
557 +
558 + for shard_id, shard_offset, shard_size in shard_offsets:
559 + # Special case for Quantization.
560 + # If quantized, we need to adjust the offset and size to account
561 + # for the packing.
562 + if isinstance(param, PackedvLLMParameter
563 + ) and param.packed_dim == param.output_dim:
564 + param.adjust_shard_indexes_for_packing(
565 + shard_size=shard_size, shard_offset=shard_offset)
566 +
567 + loaded_weight_shard = loaded_weight.narrow(param.output_dim,
568 + shard_offset,
569 + shard_size)
570 + self.weight_loader_v2(param, loaded_weight_shard, shard_id)
571 +
572 + def weight_loader_v2(self,
573 + param: BasevLLMParameter,
574 + loaded_weight: torch.Tensor,
575 + loaded_shard_id: Optional[int] = None):
576 + param_data = param.data
577 + if loaded_shard_id is None:
578 + if param.output_dim is None:
579 + assert param_data.shape == loaded_weight.shape
580 + param_data.copy_(loaded_weight)
581 + return
582 + self._load_fused_module_from_checkpoint(param, loaded_weight)
583 + return
584 +
585 + assert loaded_shard_id < len(self.output_sizes)
586 +
587 + tp_size = get_tensor_model_parallel_world_size()
588 + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
589 + shard_size = self.output_sizes[loaded_shard_id] // tp_size
590 +
591 + param.load_merged_column_weight(loaded_weight=loaded_weight,
592 + shard_id=loaded_shard_id,
593 + shard_offset=shard_offset,
594 + shard_size=shard_size)
595 +
596
597 class QKVParallelLinear(ColumnParallelLinear):
598 """Linear layers for the attention's QKV transformation.
@@ -598,6 +664,82 @@ class QKVParallelLinear(ColumnParallelLinear):
664 quant_config=quant_config,
665 prefix=prefix)
666
667 + def _get_shard_offset_mapping(self, loaded_shard_id: str):
668 + shard_offset_mapping = {
669 + "q": 0,
670 + "k": self.num_heads * self.head_size,
671 + "v": (self.num_heads + self.num_kv_heads) * self.head_size,
672 + "total": (self.num_heads + 2 * self.num_kv_heads) * self.head_size
673 + }
674 + return shard_offset_mapping.get(loaded_shard_id)
675 +
676 + def _get_shard_size_mapping(self, loaded_shard_id: str):
677 + shard_size_mapping = {
678 + "q": self.num_heads * self.head_size,
679 + "k": self.num_kv_heads * self.head_size,
680 + "v": self.num_kv_heads * self.head_size,
681 + }
682 + return shard_size_mapping.get(loaded_shard_id)
683 +
684 + def _load_fused_module_from_checkpoint(self, param: BasevLLMParameter,
685 + loaded_weight: torch.Tensor):
686 + """
687 + Handle special case for models where QKV layers are already
688 + fused on disk. In this case, we have no shard id. This function
689 + determmines the shard id by splitting these layers and then calls
690 + the weight loader using the shard id.
691 +
692 + An example of a model with these fused layers:
693 + https://huggingface.co/microsoft/Phi-3-mini-4k-instruct
694 + """
695 + shard_offsets = [
696 + # (shard_id, shard_offset, shard_size)
697 + ("q", 0, self.total_num_heads * self.head_size),
698 + ("k", self.total_num_heads * self.head_size,
699 + self.total_num_kv_heads * self.head_size),
700 + ("v",
701 + (self.total_num_heads + self.total_num_kv_heads) * self.head_size,
702 + self.total_num_kv_heads * self.head_size),
703 + ]
704 +
705 + for shard_id, shard_offset, shard_size in shard_offsets:
706 + # Special case for Quantization.
707 + # If quantized, we need to adjust the offset and size to account
708 + # for the packing.
709 + if isinstance(param, PackedvLLMParameter
710 + ) and param.packed_dim == param.output_dim:
711 + param.adjust_shard_indexes_for_packing(
712 + shard_size=shard_size, shard_offset=shard_offset)
713 +
714 + loaded_weight_shard = loaded_weight.narrow(param.output_dim,
715 + shard_offset,
716 + shard_size)
717 + self.weight_loader_v2(param, loaded_weight_shard, shard_id)
718 +
719 + def weight_loader_v2(self,
720 + param: BasevLLMParameter,
721 + loaded_weight: torch.Tensor,
722 + loaded_shard_id: Optional[str] = None):
723 + param_data = param.data
724 + if loaded_shard_id is None: # special case for certain models
725 + if param.output_dim is None:
726 + assert param_data.shape == loaded_weight.shape
727 + param_data.copy_(loaded_weight)
728 + return
729 + self._load_fused_module_from_checkpoint(param, loaded_weight)
730 + return
731 +
732 + assert loaded_shard_id in ["q", "k", "v"]
733 +
734 + shard_offset = self._get_shard_offset_mapping(loaded_shard_id)
735 + shard_size = self._get_shard_size_mapping(loaded_shard_id)
736 +
737 + param.load_qkv_weight(loaded_weight=loaded_weight,
738 + num_heads=self.num_kv_head_replicas,
739 + shard_id=loaded_shard_id,
740 + shard_offset=shard_offset,
741 + shard_size=shard_size)
742 +
743 def weight_loader(self,
744 param: Parameter,
745 loaded_weight: torch.Tensor,
@@ -798,6 +940,7 @@ class RowParallelLinear(LinearBase):
940 self.tp_size = get_tensor_model_parallel_world_size()
941 self.input_size_per_partition = divide(input_size, self.tp_size)
942 assert self.quant_method is not None
943 +
944 self.quant_method.create_weights(
945 layer=self,
946 input_size_per_partition=self.input_size_per_partition,
@@ -805,7 +948,9 @@ class RowParallelLinear(LinearBase):
948 input_size=self.input_size,
949 output_size=self.output_size,
950 params_dtype=self.params_dtype,
808 - weight_loader=self.weight_loader,
951 + weight_loader=(
952 + self.weight_loader_v2 if self.quant_method.__class__.__name__
953 + in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader),
954 prefix=prefix)
955 if not reduce_results and (bias and not skip_bias_add):
956 raise ValueError("When not reduce the results, adding bias to the "
@@ -850,6 +995,10 @@ class RowParallelLinear(LinearBase):
995 assert param_data.shape == loaded_weight.shape
996 param_data.copy_(loaded_weight)
997
998 + def weight_loader_v2(self, param: BasevLLMParameter,
999 + loaded_weight: torch.Tensor):
1000 + param.load_row_parallel_weight(loaded_weight=loaded_weight)
1001 +
1002 def forward(self, input_):
1003 if self.input_is_parallel:
1004 input_parallel = input_
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+9 -12
@@ -19,6 +19,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
19 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
20 from vllm.platforms import current_platform
21
22 +__all__ = ["CompressedTensorsLinearMethod"]
23 +
24
25 class CompressedTensorsConfig(QuantizationConfig):
26
@@ -146,18 +148,15 @@ class CompressedTensorsConfig(QuantizationConfig):
148 if weight_quant is None or input_quant is None:
149 return False
150
149 - # Confirm we have floating points.
150 - if not (weight_quant.type == QuantizationType.FLOAT
151 - and input_quant.type == QuantizationType.FLOAT):
152 - return False
153 -
151 # Confirm weight scheme is supported.
152 + is_floating_point = (weight_quant.type == QuantizationType.FLOAT
153 + and input_quant.type == QuantizationType.FLOAT)
154 is_symmetric_weight = weight_quant.symmetric
155 is_static_weight = not weight_quant.dynamic
156 is_per_tensor_or_channel_weight = (weight_quant.strategy in [
157 QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL
158 ])
160 - if not (is_symmetric_weight and is_static_weight
159 + if not (is_floating_point and is_symmetric_weight and is_static_weight
160 and is_per_tensor_or_channel_weight):
161 return False
162
@@ -169,11 +168,7 @@ class CompressedTensorsConfig(QuantizationConfig):
168 is_symmetric_activation = input_quant.symmetric
169 is_per_tensor_activation = (
170 input_quant.strategy == QuantizationStrategy.TENSOR)
172 - if not (is_symmetric_activation and is_per_tensor_activation):
173 - return False
174 -
175 - # All conditions satisfied.
176 - return True
171 + return is_symmetric_activation and is_per_tensor_activation
172
173 def _is_fp8_w8a16(self, weight_quant: BaseModel,
174 input_quant: BaseModel) -> bool:
@@ -230,6 +225,7 @@ class CompressedTensorsConfig(QuantizationConfig):
225 group_size=weight_quant.group_size)
226
227 # Detect If Activation Quantization.
228 + # TODO @dsikka: clean-up conditions
229 if is_activation_quantization_format(self.quant_format):
230 if self._is_fp8_w8a8(weight_quant, input_quant):
231 is_fp8_w8a8_supported = self._check_scheme_supported(
@@ -237,7 +233,8 @@ class CompressedTensorsConfig(QuantizationConfig):
233 if is_fp8_w8a8_supported:
234 return CompressedTensorsW8A8Fp8(
235 strategy=weight_quant.strategy,
240 - is_static_input_scheme=(not input_quant.dynamic))
236 + is_static_input_scheme=(input_quant
237 + and not input_quant.dynamic))
238 else:
239 return CompressedTensorsW8A16Fp8(
240 strategy=weight_quant.strategy,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
+11 -9
@@ -2,11 +2,10 @@ from typing import Callable, List, Optional
2
3 import torch
4 import torch.nn.functional as F
5 -from torch.nn import Parameter
5
6 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 CompressedTensorsScheme)
9 -from vllm.model_executor.utils import set_weight_attrs
8 +from vllm.model_executor.parameter import ModelWeightParameter
9
10 __all__ = ["CompressedTensorsUnquantized"]
11
@@ -24,7 +23,9 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
23 return 70
24
25 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
27 - pass
26 + # required by torch.compile to be torch.nn.Parameter
27 + layer.weight = torch.nn.Parameter(layer.weight.data,
28 + requires_grad=False)
29
30 def create_weights(self, layer: torch.nn.Module,
31 output_partition_sizes: List[int],
@@ -32,14 +33,15 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
33 params_dtype: torch.dtype, weight_loader: Callable,
34 **kwargs):
35
35 - weight = Parameter(torch.empty(sum(output_partition_sizes),
36 - input_size_per_partition,
37 - dtype=params_dtype),
38 - requires_grad=False)
36 + weight = ModelWeightParameter(data=torch.empty(
37 + sum(output_partition_sizes),
38 + input_size_per_partition,
39 + dtype=params_dtype),
40 + input_dim=1,
41 + output_dim=0,
42 + weight_loader=weight_loader)
43
40 - set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
44 layer.register_parameter("weight", weight)
42 - set_weight_attrs(weight, {"weight_loader": weight_loader})
45
46 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
47 bias: Optional[torch.Tensor]) -> torch.Tensor:
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+53 -59
@@ -8,7 +8,10 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
9 from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
10 GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N)
11 -from vllm.model_executor.utils import set_weight_attrs
11 +from vllm.model_executor.parameter import (BasevLLMParameter,
12 + ChannelQuantScaleParameter,
13 + GroupQuantScaleParameter,
14 + PackedvLLMParameter)
15 from vllm.scalar_type import scalar_types
16
17 __all__ = ["CompressedTensorsW4A16Sparse24"]
@@ -45,7 +48,12 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
48 return 80
49
50 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
48 - pass
51 + # required by torch.compile to be torch.nn.Parameter
52 + layer.weight_packed = Parameter(layer.weight_packed.data,
53 + requires_grad=False)
54 + layer.scale_packed = Parameter(layer.scale_packed.data,
55 + requires_grad=False)
56 + layer.meta = Parameter(layer.meta.data, requires_grad=False)
57
58 def create_weights(self, layer: torch.nn.Module, input_size: int,
59 output_partition_sizes: List[int],
@@ -56,79 +64,65 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
64 pack_factor = 32 // self.quant_type.size_bits
65 output_size_per_partition = sum(output_partition_sizes)
66
59 - qweight = Parameter(
60 - torch.empty(
61 - input_size_per_partition // self.tile_size // 2,
62 - output_size_per_partition * self.tile_size // pack_factor,
63 - dtype=torch.int32,
64 - ),
65 - requires_grad=False,
66 - )
67 - set_weight_attrs(
68 - qweight,
69 - {
70 - "input_dim": 0,
71 - "output_dim": 1,
72 - "packed_dim": 1,
73 - "pack_factor": pack_factor,
74 - "marlin_tile_size": self.tile_size,
75 - "weight_loader": weight_loader
76 - },
77 - )
78 -
79 - layer.register_parameter("weight_packed", qweight)
67 + qweight = PackedvLLMParameter(data=torch.empty(
68 + input_size_per_partition // self.tile_size // 2,
69 + output_size_per_partition * self.tile_size // pack_factor,
70 + dtype=torch.int32,
71 + ),
72 + input_dim=0,
73 + output_dim=1,
74 + packed_dim=1,
75 + packed_factor=pack_factor,
76 + marlin_tile_size=self.tile_size,
77 + weight_loader=weight_loader)
78
79 input_groups = (1 if self.group_size is None else
80 input_size_per_partition // self.group_size)
81
84 - scales = Parameter(
82 + weight_scale_args = {
83 + "data":
84 torch.empty(
85 input_groups,
86 output_size_per_partition,
87 dtype=params_dtype,
88 ),
90 - requires_grad=False,
91 - )
92 - set_weight_attrs(
93 - scales,
94 - {
95 - "output_dim": 1,
96 - "input_dim": None if input_groups == 1 else 0,
97 - "weight_loader": weight_loader
98 - },
99 - )
100 - layer.register_parameter("scale_packed", scales)
101 -
102 - weight_shape = Parameter(torch.empty(2, dtype=torch.int64),
103 - requires_grad=False)
89 + "weight_loader":
90 + weight_loader
91 + }
92 +
93 + if self.group_size is not None:
94 + scales = GroupQuantScaleParameter(output_dim=1,
95 + input_dim=0,
96 + **weight_scale_args)
97 + else:
98 + scales = ChannelQuantScaleParameter(output_dim=1,
99 + **weight_scale_args)
100 +
101 + weight_shape = BasevLLMParameter(data=torch.empty(2,
102 + dtype=torch.int64),
103 + weight_loader=weight_loader)
104 +
105 + meta = PackedvLLMParameter(data=torch.empty(
106 + input_size_per_partition // 8 // 2 // 2,
107 + output_size_per_partition * 2,
108 + dtype=torch.int16,
109 + ),
110 + input_dim=0,
111 + output_dim=1,
112 + packed_dim=1,
113 + packed_factor=1,
114 + marlin_tile_size=2,
115 + weight_loader=weight_loader)
116
117 + layer.register_parameter("weight_packed", qweight)
118 layer.register_parameter("weight_shape", weight_shape)
106 - set_weight_attrs(weight_shape, {"weight_loader": weight_loader})
107 -
108 - meta = Parameter(
109 - torch.empty(
110 - input_size_per_partition // 8 // 2 // 2,
111 - output_size_per_partition * 2,
112 - dtype=torch.int16,
113 - ),
114 - requires_grad=False,
115 - )
116 - set_weight_attrs(
117 - meta,
118 - {
119 - "input_dim": 0,
120 - "packed_dim": 1,
121 - "pack_factor": 1,
122 - "output_dim": 1,
123 - "marlin_tile_size": 2,
124 - "weight_loader": weight_loader
125 - },
126 - )
119 + layer.register_parameter("scale_packed", scales)
120 layer.register_parameter("meta", meta)
121
122 max_workspace_size = (
123 output_size_per_partition //
124 GPTQ_MARLIN_24_MIN_THREAD_N) * GPTQ_MARLIN_24_MAX_PARALLEL
125 +
126 workspace = Parameter(torch.zeros(max_workspace_size, dtype=torch.int),
127 requires_grad=False)
128 layer.workspace = workspace
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+32 -19
@@ -9,9 +9,10 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
10 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
11 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12 - convert_to_channelwise, create_per_channel_scale_param,
13 - create_per_tensor_scale_param)
14 -from vllm.model_executor.utils import set_weight_attrs
12 + convert_to_channelwise)
13 +from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
14 + ModelWeightParameter,
15 + PerTensorScaleParameter)
16
17 __all__ = ["CompressedTensorsW8A16Fp8"]
18
@@ -40,11 +41,19 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
41 layer.logical_widths)
42 layer.weight_scale = torch.nn.Parameter(ws_channelwise,
43 requires_grad=False)
44 + else:
45 + # required by torch.compile to be torch.nn.Parameter
46 + layer.weight_scale = torch.nn.Parameter(layer.weight_scale.data,
47 + requires_grad=False)
48
49 # Weights must be transposed for marlin
50 layer.weight = torch.nn.Parameter(layer.weight.t(),
51 requires_grad=False)
52
53 + if self.is_static_input_scheme:
54 + # required by torch.compile to be torch.nn.Parameter
55 + layer.input_scale = torch.nn.Parameter(layer.input_scale.data,
56 + requires_grad=False)
57 prepare_fp8_layer_for_marlin(layer, strategy="channel")
58
59 def create_weights(self, layer: torch.nn.Module, input_size: int,
@@ -60,35 +69,39 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
69 layer.orig_dtype = params_dtype
70
71 # WEIGHT
63 - weight = torch.nn.Parameter(torch.empty(output_size_per_partition,
64 - input_size_per_partition,
65 - dtype=torch.float8_e4m3fn),
66 - requires_grad=False)
72 + weight = ModelWeightParameter(data=torch.empty(
73 + output_size_per_partition,
74 + input_size_per_partition,
75 + dtype=torch.float8_e4m3fn),
76 + input_dim=1,
77 + output_dim=0,
78 + weight_loader=weight_loader)
79 layer.register_parameter("weight", weight)
68 - set_weight_attrs(weight, {
69 - "input_dim": 1,
70 - "output_dim": 0,
71 - "weight_loader": weight_loader,
72 - })
80
81 # WEIGHT SCALE
75 - layer_kwargs = {"weight_loader": weight_loader}
82 if self.strategy == QuantizationStrategy.CHANNEL:
77 - weight_scale = create_per_channel_scale_param(
78 - output_partition_sizes, **layer_kwargs)
83 + weight_scale = ChannelQuantScaleParameter(
84 + data=torch.empty((sum(output_partition_sizes), 1),
85 + dtype=torch.float32),
86 + output_dim=0,
87 + weight_loader=weight_loader)
88 elif self.strategy == QuantizationStrategy.TENSOR:
80 - weight_scale = create_per_tensor_scale_param(
81 - output_partition_sizes, **layer_kwargs)
89 + weight_scale = PerTensorScaleParameter(data=torch.empty(
90 + len(output_partition_sizes), dtype=torch.float32),
91 + weight_loader=weight_loader)
92 else:
93 raise ValueError(
94 f"Unsupported weight strategy={self.strategy}, "
95 f"supported strategies are {SUPPORTED_STRATEGIES}")
96 +
97 + weight_scale[:] = torch.finfo(torch.float32).min
98 layer.register_parameter("weight_scale", weight_scale)
99
100 # INPUT SCALE (to deal with converted checkpoints)
101 if self.is_static_input_scheme:
90 - input_scale = create_per_tensor_scale_param(
91 - output_partition_sizes, **layer_kwargs)
102 + input_scale = PerTensorScaleParameter(data=torch.empty(
103 + len(output_partition_sizes), dtype=torch.float32),
104 + weight_loader=weight_loader)
105 layer.register_parameter("input_scale", input_scale)
106
107 def apply_weights(self,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+31 -20
@@ -8,10 +8,10 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 QuantizationStrategy)
10 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11 - apply_fp8_linear, create_per_channel_scale_param,
12 - create_per_tensor_scale_param, cutlass_fp8_supported,
13 - requantize_with_max_scale)
14 -from vllm.model_executor.utils import set_weight_attrs
11 + apply_fp8_linear, cutlass_fp8_supported, requantize_with_max_scale)
12 +from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
13 + ModelWeightParameter,
14 + PerTensorScaleParameter)
15
16 __all__ = ["CompressedTensorsW8A8Fp8"]
17
@@ -46,6 +46,9 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
46 elif self.strategy == QuantizationStrategy.CHANNEL:
47 weight = layer.weight
48 layer.weight = Parameter(weight.t(), requires_grad=False)
49 + # required by torch.compile to be torch.nn.Parameter
50 + layer.weight_scale = Parameter(layer.weight_scale.data,
51 + requires_grad=False)
52
53 else:
54 raise ValueError(f"Unknown quantization strategy {self.strategy}")
@@ -66,32 +69,40 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
69 layer.logical_widths = output_partition_sizes
70
71 # WEIGHT
69 - weight = torch.nn.Parameter(torch.empty(output_size_per_partition,
70 - input_size_per_partition,
71 - dtype=torch.float8_e4m3fn),
72 - requires_grad=False)
72 + weight = ModelWeightParameter(data=torch.empty(
73 + output_size_per_partition,
74 + input_size_per_partition,
75 + dtype=torch.float8_e4m3fn),
76 + input_dim=1,
77 + output_dim=0,
78 + weight_loader=weight_loader)
79 layer.register_parameter("weight", weight)
74 - set_weight_attrs(weight, {
75 - "input_dim": 1,
76 - "output_dim": 0,
77 - "weight_loader": weight_loader,
78 - })
80
81 # WEIGHT SCALE
81 - layer_kwargs = {"weight_loader": weight_loader}
82 + # TODO: update create_xxx_parameter functions to return
83 + # the newly added parameters
84 if self.strategy == QuantizationStrategy.CHANNEL:
83 - weight_scale = create_per_channel_scale_param(
84 - output_partition_sizes, **layer_kwargs)
85 + weight_scale = ChannelQuantScaleParameter(
86 + data=torch.empty((sum(output_partition_sizes), 1),
87 + dtype=torch.float32),
88 + output_dim=0,
89 + weight_loader=weight_loader)
90 else:
91 assert self.strategy == QuantizationStrategy.TENSOR
87 - weight_scale = create_per_tensor_scale_param(
88 - output_partition_sizes, **layer_kwargs)
92 + weight_scale = PerTensorScaleParameter(data=torch.empty(
93 + len(output_partition_sizes), dtype=torch.float32),
94 + weight_loader=weight_loader)
95 +
96 + # min requirement for fp8 kernels
97 + weight_scale[:] = torch.finfo(torch.float32).min
98 layer.register_parameter("weight_scale", weight_scale)
99
100 # INPUT SCALE
101 if self.is_static_input_scheme:
93 - input_scale = create_per_tensor_scale_param(
94 - output_partition_sizes, **layer_kwargs)
102 + input_scale = PerTensorScaleParameter(data=torch.empty(
103 + len(output_partition_sizes), dtype=torch.float32),
104 + weight_loader=weight_loader)
105 + input_scale[:] = torch.finfo(torch.float32).min
106 layer.register_parameter("input_scale", input_scale)
107
108 def apply_weights(self,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+27 -20
@@ -8,9 +8,11 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 QuantizationStrategy)
10 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11 - apply_int8_linear, convert_to_channelwise, create_per_channel_scale_param,
12 - create_per_tensor_scale_param)
13 -from vllm.model_executor.utils import set_weight_attrs
11 + apply_int8_linear, convert_to_channelwise)
12 +from vllm.model_executor.parameter import (BasevLLMParameter,
13 + ChannelQuantScaleParameter,
14 + ModelWeightParameter,
15 + PerTensorScaleParameter)
16
17
18 class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
@@ -39,7 +41,9 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
41 ws_channelwise = convert_to_channelwise(layer.weight_scale,
42 self.logical_widths)
43 layer.weight_scale = Parameter(ws_channelwise, requires_grad=False)
42 -
44 + else:
45 + layer.weight_scale = Parameter(layer.weight_scale.data,
46 + requires_grad=False)
47 # INPUT SCALE
48 if self.is_static_input_scheme:
49 layer.input_scale = Parameter(layer.input_scale.max(),
@@ -55,32 +59,35 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
59 self.logical_widths = output_partition_sizes
60
61 # WEIGHT
58 - weight = Parameter(torch.empty(sum(output_partition_sizes),
59 - input_size_per_partition,
60 - dtype=torch.int8),
61 - requires_grad=False)
62 + weight = ModelWeightParameter(data=torch.empty(
63 + sum(output_partition_sizes),
64 + input_size_per_partition,
65 + dtype=torch.int8),
66 + input_dim=1,
67 + output_dim=0,
68 + weight_loader=weight_loader)
69 +
70 layer.register_parameter("weight", weight)
63 - set_weight_attrs(weight, {
64 - "input_dim": 1,
65 - "output_dim": 0,
66 - "weight_loader": weight_loader,
67 - })
71
72 # WEIGHT SCALE
70 - layer_kwargs = {"weight_loader": weight_loader}
73 if self.strategy == QuantizationStrategy.CHANNEL:
72 - weight_scale = create_per_channel_scale_param(
73 - output_partition_sizes, **layer_kwargs)
74 + weight_scale = ChannelQuantScaleParameter(
75 + data=torch.empty((sum(output_partition_sizes), 1),
76 + dtype=torch.float32),
77 + output_dim=0,
78 + weight_loader=weight_loader)
79 else:
80 assert self.strategy == QuantizationStrategy.TENSOR
76 - weight_scale = create_per_tensor_scale_param(
77 - output_partition_sizes, **layer_kwargs)
81 + weight_scale = PerTensorScaleParameter(data=torch.empty(
82 + len(output_partition_sizes), dtype=torch.float32),
83 + weight_loader=weight_loader)
84 layer.register_parameter("weight_scale", weight_scale)
85
86 # INPUT SCALE
87 if self.is_static_input_scheme:
82 - input_scale = create_per_tensor_scale_param(
83 - output_partition_sizes, **layer_kwargs)
88 + input_scale = BasevLLMParameter(data=torch.empty(
89 + 1, dtype=torch.float32),
90 + weight_loader=weight_loader)
91 layer.register_parameter("input_scale", input_scale)
92
93 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+48 -54
@@ -1,7 +1,6 @@
1 from typing import Callable, List, Optional
2
3 import torch
4 -from torch.nn import Parameter
4
5 from vllm import _custom_ops as ops
6 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
@@ -10,7 +9,10 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
9 apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
10 marlin_permute_scales, replace_tensor, verify_marlin_supported,
11 verify_marlin_supports_shape)
13 -from vllm.model_executor.utils import set_weight_attrs
12 +from vllm.model_executor.parameter import (BasevLLMParameter,
13 + ChannelQuantScaleParameter,
14 + GroupQuantScaleParameter,
15 + PackedvLLMParameter)
16 from vllm.scalar_type import scalar_types
17
18 __all__ = ["CompressedTensorsWNA16"]
@@ -30,17 +32,12 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
32
33 self.pack_factor = 32 // num_bits
34 self.strategy = strategy
35 + self.group_size = -1 if group_size is None else group_size
36
34 - self.group_size: int
35 - if group_size is None:
36 - if self.strategy != "channel":
37 - raise ValueError(
38 - "Marlin kernels require group quantization or "
39 - "channelwise quantization, but found no group "
40 - "size and strategy is not channelwise.")
41 - self.group_size = -1
42 - else:
43 - self.group_size = group_size
37 + if self.group_size == -1 and self.strategy != "channel":
38 + raise ValueError("Marlin kernels require group quantization or "
39 + "channelwise quantization, but found no group "
40 + "size and strategy is not channelwise.")
41
42 if num_bits not in WNA16_SUPPORTED_TYPES_MAP:
43 raise ValueError(
@@ -63,11 +60,12 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
60 input_size_per_partition: int,
61 params_dtype: torch.dtype, weight_loader: Callable,
62 **kwargs):
63 +
64 output_size_per_partition = sum(output_partition_sizes)
65
66 # If group_size is -1, we are in channelwise case.
67 channelwise = (self.group_size == -1)
70 - group_size = input_size if channelwise else self.group_size
68 + group_size = self.group_size if self.group_size != -1 else input_size
69 row_parallel = (input_size != input_size_per_partition)
70 # In the case of channelwise quantization, we need to replicate the
71 # scales across all gpus.
@@ -79,60 +77,51 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
77 input_size=input_size,
78 group_size=group_size)
79
82 - weight_scale_dim = None
80 scales_and_zp_size = input_size // group_size
81
82 if partition_scales:
83 assert input_size_per_partition % group_size == 0
87 - weight_scale_dim = 1
84 scales_and_zp_size = input_size_per_partition // group_size
85
90 - weight = Parameter(
91 - torch.empty(
92 - output_size_per_partition,
93 - input_size_per_partition // self.pack_factor,
94 - dtype=torch.int32,
95 - ),
96 - requires_grad=False,
97 - )
98 -
99 - set_weight_attrs(
100 - weight, {
101 - "input_dim": 1,
102 - "output_dim": 0,
103 - "packed_dim": 1,
104 - "pack_factor": self.pack_factor,
105 - "weight_loader": weight_loader
106 - })
107 - layer.register_parameter("weight_packed", weight)
108 -
109 - weight_scale = Parameter(
86 + weight = PackedvLLMParameter(input_dim=1,
87 + output_dim=0,
88 + weight_loader=weight_loader,
89 + packed_factor=self.pack_factor,
90 + packed_dim=1,
91 + data=torch.empty(
92 + output_size_per_partition,
93 + input_size_per_partition //
94 + self.pack_factor,
95 + dtype=torch.int32,
96 + ))
97 +
98 + weight_scale_args = {
99 + "weight_loader":
100 + weight_loader,
101 + "data":
102 torch.empty(
103 output_size_per_partition,
104 scales_and_zp_size,
105 dtype=params_dtype,
114 - ),
115 - requires_grad=False,
116 - )
117 -
118 - set_weight_attrs(
119 - weight_scale, {
120 - "weight_loader": weight_loader,
121 - "input_dim": weight_scale_dim,
122 - "output_dim": 0
123 - })
124 - layer.register_parameter("weight_scale", weight_scale)
106 + )
107 + }
108 + if self.group_size == -1:
109 + weight_scale = ChannelQuantScaleParameter(output_dim=0,
110 + **weight_scale_args)
111 + else:
112 + weight_scale = GroupQuantScaleParameter(output_dim=0,
113 + input_dim=1,
114 + **weight_scale_args)
115
116 # A 2D array defining the original shape of the weights
117 # before packing
128 - weight_shape = Parameter(torch.empty(2, dtype=torch.int64),
129 - requires_grad=False)
118 + weight_shape = BasevLLMParameter(data=torch.empty(2,
119 + dtype=torch.int64),
120 + weight_loader=weight_loader)
121
122 + layer.register_parameter("weight_packed", weight)
123 + layer.register_parameter("weight_scale", weight_scale)
124 layer.register_parameter("weight_shape", weight_shape)
132 - set_weight_attrs(weight_shape, {
133 - "weight_loader": weight_loader,
134 - "ignore_warning": True,
135 - })
125
126 layer.input_size_per_partition = input_size_per_partition
127 layer.output_size_per_partition = output_size_per_partition
@@ -154,10 +143,15 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
143
144 # No zero-point
145 layer.weight_zp = marlin_make_empty_g_idx(device)
146 + # Update for kernel
147 + layer.weight_packed = torch.nn.Parameter(
148 + layer.weight_packed.t().contiguous(), requires_grad=False)
149 + layer.weight_scale = torch.nn.Parameter(
150 + layer.weight_scale.squeeze().t().contiguous(), requires_grad=False)
151
152 # Repack weights from compressed-tensors format to marlin format.
153 marlin_qweight = ops.gptq_marlin_repack(
160 - layer.weight_packed.t().contiguous(),
154 + layer.weight_packed,
155 perm=layer.g_idx_sort_indices,
156 size_k=layer.input_size_per_partition,
157 size_n=layer.output_size_per_partition,
@@ -166,7 +160,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
160
161 # Permute scales from compressed-tensors format to marlin format.
162 marlin_scales = marlin_permute_scales(
169 - layer.weight_scale.squeeze().t().contiguous(),
163 + layer.weight_scale,
164 size_k=layer.input_size_per_partition,
165 size_n=layer.output_size_per_partition,
166 group_size=layer.group_size)
vllm/model_executor/parameter.py new
+277
@@ -0,0 +1,277 @@
1 +from typing import Callable, Optional, Union
2 +
3 +import torch
4 +from torch.nn import Parameter
5 +
6 +from vllm.distributed import get_tensor_model_parallel_rank
7 +from vllm.logger import init_logger
8 +
9 +__all__ = [
10 + "BasevLLMParameter", "PackedvLLMParameter", "PerTensorScaleParameter",
11 + "ModelWeightParameter", "ChannelQuantScaleParameter",
12 + "GroupQuantScaleParameter"
13 +]
14 +
15 +logger = init_logger(__name__)
16 +
17 +
18 +class BasevLLMParameter(Parameter):
19 + """
20 + Base parameter for vLLM linear layers. Extends the torch.nn.parameter
21 + by taking in a linear weight loader. Will copy the loaded weight
22 + into the parameter when the provided weight loader is called.
23 + """
24 +
25 + def __new__(cls, data: torch.Tensor, **kwargs):
26 +
27 + return super().__new__(cls, data=data, requires_grad=False)
28 +
29 + def __init__(self, data: torch.Tensor, weight_loader: Callable):
30 + """
31 + Initialize the BasevLLMParameter
32 +
33 + :param data: torch tensor with the parameter data
34 + :param weight_loader: weight loader callable
35 +
36 + :returns: a torch.nn.parameter
37 + """
38 +
39 + self._weight_loader = weight_loader
40 +
41 + @property
42 + def weight_loader(self):
43 + return self._weight_loader
44 +
45 + def _assert_and_load(self, loaded_weight: torch.Tensor):
46 + assert self.data.shape == loaded_weight.shape
47 + self.data.copy_(loaded_weight)
48 +
49 + def load_column_parallel_weight(self, loaded_weight: torch.Tensor):
50 + self._assert_and_load(loaded_weight)
51 +
52 + def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
53 + self._assert_and_load(loaded_weight)
54 +
55 + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
56 + self._assert_and_load(loaded_weight)
57 +
58 + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
59 + self._assert_and_load(loaded_weight)
60 +
61 +
62 +class _ColumnvLLMParameter(BasevLLMParameter):
63 + """
64 + Private class defining weight loading functionality
65 + (load_merged_column_weight, load_qkv_weight)
66 + for parameters being loaded into linear layers with column
67 + parallelism. This includes QKV and MLP layers which are
68 + not already fused on disk. Requires an output dimension
69 + to be defined. Called within the weight loader of
70 + each of the column parallel linear layers.
71 + """
72 +
73 + def __init__(self, output_dim: int, **kwargs):
74 + self._output_dim = output_dim
75 + super().__init__(**kwargs)
76 +
77 + @property
78 + def output_dim(self):
79 + return self._output_dim
80 +
81 + def load_column_parallel_weight(self, loaded_weight: torch.Tensor):
82 + tp_rank = get_tensor_model_parallel_rank()
83 + shard_size = self.data.shape[self.output_dim]
84 + loaded_weight = loaded_weight.narrow(self.output_dim,
85 + tp_rank * shard_size, shard_size)
86 + assert self.data.shape == loaded_weight.shape
87 + self.data.copy_(loaded_weight)
88 +
89 + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
90 +
91 + shard_offset = kwargs.get("shard_offset")
92 + shard_size = kwargs.get("shard_size")
93 + if isinstance(
94 + self,
95 + PackedvLLMParameter) and self.packed_dim == self.output_dim:
96 + shard_size, shard_offset = self.adjust_shard_indexes_for_packing(
97 + shard_offset=shard_offset, shard_size=shard_size)
98 +
99 + param_data = self.data
100 +
101 + tp_rank = get_tensor_model_parallel_rank()
102 + param_data = param_data.narrow(self.output_dim, shard_offset,
103 + shard_size)
104 + loaded_weight = loaded_weight.narrow(self.output_dim,
105 + tp_rank * shard_size, shard_size)
106 + assert param_data.shape == loaded_weight.shape
107 + param_data.copy_(loaded_weight)
108 +
109 + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
110 +
111 + shard_offset = kwargs.get("shard_offset")
112 + shard_size = kwargs.get("shard_size")
113 + shard_id = kwargs.get("shard_id")
114 + num_heads = kwargs.get("num_heads")
115 +
116 + if isinstance(
117 + self,
118 + PackedvLLMParameter) and self.output_dim == self.packed_dim:
119 + shard_size, shard_offset = self.adjust_shard_indexes_for_packing(
120 + shard_offset=shard_offset, shard_size=shard_size)
121 +
122 + param_data = self.data
123 + tp_rank = get_tensor_model_parallel_rank()
124 + shard_id = tp_rank if shard_id == "q" else tp_rank // num_heads
125 + param_data = param_data.narrow(self.output_dim, shard_offset,
126 + shard_size)
127 + loaded_weight = loaded_weight.narrow(self.output_dim,
128 + shard_id * shard_size, shard_size)
129 +
130 + assert param_data.shape == loaded_weight.shape
131 + param_data.copy_(loaded_weight)
132 +
133 +
134 +class ModelWeightParameter(_ColumnvLLMParameter):
135 + """
136 + Parameter class for linear layer weights. Extends the
137 + _ColumnvLLMParameter by adding loading functionality
138 + for linear layers with row parallel functionality.
139 + Requires an input dimension to be defined.
140 + """
141 +
142 + def __init__(self, input_dim: int, **kwargs):
143 + self._input_dim = input_dim
144 + super().__init__(**kwargs)
145 +
146 + @property
147 + def input_dim(self):
148 + return self._input_dim
149 +
150 + def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
151 + tp_rank = get_tensor_model_parallel_rank()
152 + shard_size = self.data.shape[self.input_dim]
153 + loaded_weight = loaded_weight.narrow(self.input_dim,
154 + tp_rank * shard_size, shard_size)
155 +
156 + if len(loaded_weight.shape) == 0:
157 + loaded_weight = loaded_weight.reshape(1)
158 +
159 + assert self.data.shape == loaded_weight.shape
160 + self.data.copy_(loaded_weight)
161 +
162 +
163 +class GroupQuantScaleParameter(ModelWeightParameter):
164 + """
165 + Parameter class for weight scales loaded for weights with
166 + grouped quantization. Equivalent to ModelWeightParameter.
167 + """
168 + pass
169 +
170 +
171 +class ChannelQuantScaleParameter(_ColumnvLLMParameter):
172 + """
173 + Parameter class for weight scales loaded for weights with
174 + channel-wise quantization. Equivalent to _ColumnvLLMParameter.
175 + """
176 + pass
177 +
178 +
179 +class PerTensorScaleParameter(BasevLLMParameter):
180 + """
181 + Parameter class for scales where the number of scales is
182 + equivalent to the number of logical matrices in fused linear
183 + layers (e.g. for QKV, there are 3 scales loaded from disk).
184 + This is relevant to weights with per-tensor quantization.
185 + Adds functionality to map the scalers to a shard during
186 + weight loading.
187 +
188 + Note: additional parameter manipulation may be handled
189 + for each quantization config specifically, within
190 + process_weights_after_loading
191 + """
192 +
193 + def __init__(self, **kwargs):
194 + self.qkv_idxs = {"q": 0, "k": 1, "v": 2}
195 + super().__init__(**kwargs)
196 +
197 + def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
198 + if isinstance(shard_id, int):
199 + return shard_id
200 +
201 + assert isinstance(shard_id, str)
202 + assert shard_id in self.qkv_idxs
203 + return self.qkv_idxs[shard_id]
204 +
205 + def load_merged_column_weight(self, *args, **kwargs):
206 + self._load_into_shard_id(*args, **kwargs)
207 +
208 + def load_qkv_weight(self, *args, **kwargs):
209 + self._load_into_shard_id(*args, **kwargs)
210 +
211 + def load_column_parallel_weight(self, *args, **kwargs):
212 + self._load_into_shard_id(*args, **kwargs)
213 +
214 + def _load_into_shard_id(self, loaded_weight: torch.Tensor,
215 + shard_id: Union[str, int], **kwargs):
216 + """
217 + Slice the parameter data based on the shard id for
218 + loading.
219 + """
220 +
221 + param_data = self.data
222 + shard_id = self._shard_id_as_int(shard_id)
223 +
224 + # AutoFP8 scales do not have a shape
225 + # compressed-tensors scales do have a shape
226 + if len(loaded_weight.shape) != 0:
227 + assert loaded_weight.shape[0] == 1
228 + loaded_weight = loaded_weight[0]
229 +
230 + param_data = param_data[shard_id]
231 + assert param_data.shape == loaded_weight.shape
232 + param_data.copy_(loaded_weight)
233 +
234 +
235 +class PackedvLLMParameter(ModelWeightParameter):
236 + """
237 + Parameter for model weights which are packed on disk.
238 + Example: GPTQ Marlin weights are int4 or int8, packed into int32.
239 + Extends the ModelWeightParameter to take in the
240 + packed factor, the packed dimension, and optionally, marlin
241 + tile size for marlin kernels. Adjusts the shard_size and
242 + shard_offset for fused linear layers model weight loading
243 + by accounting for packing and optionally, marlin tile size.
244 + """
245 +
246 + def __init__(self,
247 + packed_factor: int,
248 + packed_dim: int,
249 + marlin_tile_size: Optional[int] = None,
250 + **kwargs):
251 + self._packed_factor = packed_factor
252 + self._packed_dim = packed_dim
253 + self._marlin_tile = marlin_tile_size
254 + super().__init__(**kwargs)
255 +
256 + @property
257 + def packed_dim(self):
258 + return self._packed_dim
259 +
260 + @property
261 + def packed_factor(self):
262 + return self._packed_factor
263 +
264 + @property
265 + def marlin_tile(self):
266 + return self._marlin_tile
267 +
268 + def _adjust_shard_indexes_for_marlin(self, shard_size, shard_offset):
269 + return shard_size * self.marlin_tile, shard_offset * self.marlin_tile
270 +
271 + def adjust_shard_indexes_for_packing(self, shard_size, shard_offset):
272 + shard_size = shard_size // self.packed_factor
273 + shard_offset = shard_offset // self.packed_factor
274 + if self.marlin_tile is not None:
275 + return self._adjust_shard_indexes_for_marlin(
276 + shard_size, shard_offset)
277 + return shard_size, shard_offset