A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ Misc ] `fbgemm` checkpoints (#6559)
Robert Shaw committed
Jul 20, 2024 at 12:36 UTC
683e3cb9c4b25d10d507817cb0521883d29cc082
24 files changed
+234
-47
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
+2
-2
@@ -4,8 +4,8 @@ tasks:
4
- name: "gsm8k"
5
metrics:
6
- name: "exact_match,strict-match"
7
- value: 0.769
7
+ value: 0.752
8
- name: "exact_match,flexible-extract"
9
- value: 0.769
9
+ value: 0.754
10
limit: 1000
11
num_fewshot: 5
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform -b auto -l 1000 -f 5 -t 1
2
+model_name: "nm-testing/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.753
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.753
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1
-1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46
done
47
48
lm_eval --model vllm \
49
- --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray",trust_remote_code=true,max_model_len=4096 \
49
+ --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend="ray",trust_remote_code=true,max_model_len=4096 \
50
--tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51
--batch_size $BATCH_SIZE
vllm/_custom_ops.py
+2
@@ -315,6 +315,8 @@ def scaled_fp8_quant(
315
Args:
316
input: The input tensor to be quantized to FP8
317
scale: Optional scaling factor for the FP8 quantization
318
+ scale_ub: Optional upper bound for scaling factor in dynamic
319
+ per token case
320
batch_dim_padding: If specified, pad the first dimension
321
of the output to at least this value.
322
use_per_token_if_dynamic: Whether to do per_tensor or per_token
vllm/attention/layer.py
+2
-1
@@ -34,6 +34,7 @@ class Attention(nn.Module):
34
cache_config: Optional[CacheConfig] = None,
35
quant_config: Optional[QuantizationConfig] = None,
36
blocksparse_params: Optional[Dict[str, Any]] = None,
37
+ prefix: str = "",
38
) -> None:
39
super().__init__()
40
if cache_config is not None:
@@ -56,7 +57,7 @@ class Attention(nn.Module):
57
self._k_scale = 1.0
58
self._v_scale = 1.0
59
quant_method = quant_config.get_quant_method(
59
- self) if quant_config else None
60
+ self, prefix=prefix) if quant_config else None
61
if quant_method is not None:
62
assert isinstance(quant_method, Fp8KVCacheMethod)
63
# TODO (mgoin): kv cache dtype should be specified in the FP8
vllm/config.py
+1
-1
@@ -251,7 +251,7 @@ class ModelConfig:
251
f"supported in ROCm.")
252
if (self.quantization
253
not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin",
254
- "compressed_tensors")):
254
+ "fbgemm_fp8", "compressed_tensors")):
255
logger.warning(
256
"%s quantization is not fully "
257
"optimized yet. The speed can be slower than "
vllm/model_executor/layers/fused_moe/layer.py
+1
-1
@@ -182,7 +182,7 @@ class FusedMoE(torch.nn.Module):
182
self.quant_method: Optional[QuantizeMethodBase] = (
183
UnquantizedFusedMoEMethod())
184
else:
185
- self.quant_method = quant_config.get_quant_method(self)
185
+ self.quant_method = quant_config.get_quant_method(self, prefix)
186
assert self.quant_method is not None
187
188
self.quant_method.create_weights(
vllm/model_executor/layers/linear.py
+16
-10
@@ -141,6 +141,7 @@ class LinearBase(torch.nn.Module):
141
skip_bias_add: bool = False,
142
params_dtype: Optional[torch.dtype] = None,
143
quant_config: Optional[QuantizationConfig] = None,
144
+ prefix: str = "",
145
):
146
super().__init__()
147
@@ -155,7 +156,8 @@ class LinearBase(torch.nn.Module):
156
self.quant_method: Optional[
157
QuantizeMethodBase] = UnquantizedLinearMethod()
158
else:
158
- self.quant_method = quant_config.get_quant_method(self)
159
+ self.quant_method = quant_config.get_quant_method(self,
160
+ prefix=prefix)
161
162
def forward(self, x: torch.Tensor) -> torch.Tensor:
163
raise NotImplementedError
@@ -182,9 +184,13 @@ class ReplicatedLinear(LinearBase):
184
skip_bias_add: bool = False,
185
params_dtype: Optional[torch.dtype] = None,
186
quant_config: Optional[QuantizationConfig] = None,
185
- prefix: Optional[str] = None):
186
- super().__init__(input_size, output_size, skip_bias_add, params_dtype,
187
- quant_config)
187
+ prefix: str = ""):
188
+ super().__init__(input_size,
189
+ output_size,
190
+ skip_bias_add,
191
+ params_dtype,
192
+ quant_config,
193
+ prefix=prefix)
194
195
# All the linear layer supports quant method.
196
assert self.quant_method is not None
@@ -258,9 +264,9 @@ class ColumnParallelLinear(LinearBase):
264
params_dtype: Optional[torch.dtype] = None,
265
quant_config: Optional[QuantizationConfig] = None,
266
output_sizes: Optional[List[int]] = None,
261
- prefix: Optional[str] = None):
267
+ prefix: str = ""):
268
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
263
- quant_config)
269
+ quant_config, prefix)
270
271
self.gather_output = gather_output
272
@@ -370,7 +376,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
376
skip_bias_add: bool = False,
377
params_dtype: Optional[torch.dtype] = None,
378
quant_config: Optional[QuantizationConfig] = None,
373
- prefix: Optional[str] = None):
379
+ prefix: str = ""):
380
self.output_sizes = output_sizes
381
tp_size = get_tensor_model_parallel_world_size()
382
assert all(output_size % tp_size == 0 for output_size in output_sizes)
@@ -514,7 +520,7 @@ class QKVParallelLinear(ColumnParallelLinear):
520
skip_bias_add: bool = False,
521
params_dtype: Optional[torch.dtype] = None,
522
quant_config: Optional[QuantizationConfig] = None,
517
- prefix: Optional[str] = None):
523
+ prefix: str = ""):
524
self.hidden_size = hidden_size
525
self.head_size = head_size
526
self.total_num_heads = total_num_heads
@@ -707,9 +713,9 @@ class RowParallelLinear(LinearBase):
713
params_dtype: Optional[torch.dtype] = None,
714
reduce_results: bool = True,
715
quant_config: Optional[QuantizationConfig] = None,
710
- prefix: Optional[str] = None):
716
+ prefix: str = ""):
717
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
712
- quant_config)
718
+ quant_config, prefix)
719
720
self.input_is_parallel = input_is_parallel
721
self.reduce_results = reduce_results
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
10
CompressedTensorsConfig)
11
from vllm.model_executor.layers.quantization.deepspeedfp import (
12
DeepSpeedFPConfig)
13
+from vllm.model_executor.layers.quantization.fbgemm_fp8 import FBGEMMFp8Config
14
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
15
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
16
from vllm.model_executor.layers.quantization.gptq_marlin import (
@@ -24,6 +25,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
25
"awq": AWQConfig,
26
"deepspeedfp": DeepSpeedFPConfig,
27
"fp8": Fp8Config,
28
+ "fbgemm_fp8": FBGEMMFp8Config,
29
# The order of gptq methods is important for config.py iteration over
30
# override_quantization_method(..)
31
"marlin": MarlinConfig,
vllm/model_executor/layers/quantization/aqlm.py
+2
-2
@@ -207,8 +207,8 @@ class AQLMConfig(QuantizationConfig):
207
return cls(in_group_size, nbits_per_codebook, num_code_books,
208
out_group_size)
209
210
- def get_quant_method(
211
- self, layer: torch.nn.Module) -> Optional["AQLMLinearMethod"]:
210
+ def get_quant_method(self, layer: torch.nn.Module,
211
+ prefix: str) -> Optional["AQLMLinearMethod"]:
212
if isinstance(layer, LinearBase):
213
return AQLMLinearMethod(self)
214
return None
vllm/model_executor/layers/quantization/awq.py
+2
-2
@@ -63,8 +63,8 @@ class AWQConfig(QuantizationConfig):
63
zero_point = cls.get_from_keys(config, ["zero_point"])
64
return cls(weight_bits, group_size, zero_point)
65
66
- def get_quant_method(
67
- self, layer: torch.nn.Module) -> Optional["AWQLinearMethod"]:
66
+ def get_quant_method(self, layer: torch.nn.Module,
67
+ prefix: str) -> Optional["AWQLinearMethod"]:
68
if isinstance(layer, LinearBase):
69
return AWQLinearMethod(self)
70
return None
vllm/model_executor/layers/quantization/base_config.py
+3
-2
@@ -97,12 +97,13 @@ class QuantizationConfig(ABC):
97
return default
98
99
@abstractmethod
100
- def get_quant_method(
101
- self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
100
+ def get_quant_method(self, layer: torch.nn.Module,
101
+ prefix: str) -> Optional[QuantizeMethodBase]:
102
"""Get the quantize method to use for the quantized layer.
103
104
Args:
105
layer: The layer for the quant method.
106
+ prefix: The full name of the layer in the state dict
107
Returns:
108
The quantize method. None if the given layer doesn't support quant
109
method.
vllm/model_executor/layers/quantization/bitsandbytes.py
+2
-3
@@ -60,9 +60,8 @@ class BitsAndBytesConfig(QuantizationConfig):
60
target_modules = cls.get_from_keys(config, ["target_modules"])
61
return cls(adapter_name, target_modules)
62
63
- def get_quant_method(
64
- self,
65
- layer: torch.nn.Module) -> Optional["BitsAndBytesLinearMethod"]:
63
+ def get_quant_method(self, layer: torch.nn.Module,
64
+ prefix: str) -> Optional["BitsAndBytesLinearMethod"]:
65
if isinstance(layer, LinearBase):
66
return BitsAndBytesLinearMethod(self)
67
return None
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5
-1
@@ -44,8 +44,12 @@ class CompressedTensorsConfig(QuantizationConfig):
44
def get_name(self) -> str:
45
return "compressed_tensors"
46
47
+ # TODO (@robertgshaw2-neuralmagic): do layer skipping though here
48
+ # rather than though create_weights to match other methods
49
def get_quant_method(
48
- self, layer: torch.nn.Module
50
+ self,
51
+ layer: torch.nn.Module,
52
+ prefix: str,
53
) -> Optional["CompressedTensorsLinearMethod"]:
54
if isinstance(layer, LinearBase):
55
return CompressedTensorsLinearMethod(self)
vllm/model_executor/layers/quantization/deepspeedfp.py
+2
-3
@@ -69,9 +69,8 @@ class DeepSpeedFPConfig(QuantizationConfig):
69
"quantize_config.json",
70
]
71
72
- def get_quant_method(
73
- self,
74
- layer: torch.nn.Module) -> Optional["DeepSpeedFPLinearMethod"]:
72
+ def get_quant_method(self, layer: torch.nn.Module,
73
+ prefix: str) -> Optional["DeepSpeedFPLinearMethod"]:
74
if isinstance(layer, LinearBase):
75
return DeepSpeedFPLinearMethod(self)
76
return None
vllm/model_executor/layers/quantization/fbgemm_fp8.py
new
+158
@@ -0,0 +1,158 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+from torch.nn import Module
5
+from torch.nn.parameter import Parameter
6
+
7
+from vllm.logger import init_logger
8
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9
+ UnquantizedLinearMethod)
10
+from vllm.model_executor.layers.quantization.base_config import (
11
+ QuantizationConfig, QuantizeMethodBase)
12
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
13
+ apply_fp8_linear, create_per_channel_scale_param)
14
+from vllm.model_executor.utils import set_weight_attrs
15
+
16
+logger = init_logger(__name__)
17
+
18
+# Note: this is a hack. We should update each model to register the
19
+# stacked params and get it from there instead in a future PR.
20
+# fused_name: List[shard_name]
21
+_FUSED_LAYER_NAME_MAPPING = {
22
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
23
+ "gate_up_proj": ["gate_proj", "up_proj"]
24
+}
25
+
26
+
27
+class FBGEMMFp8Config(QuantizationConfig):
28
+ """Config class for FBGEMM Fp8."""
29
+
30
+ def __init__(self, ignore_list: List[str], input_scale_ub: float):
31
+ self.ignore_list = ignore_list
32
+ self.input_scale_ub = input_scale_ub
33
+
34
+ @classmethod
35
+ def get_name(cls) -> str:
36
+ return "fbgemm_fp8"
37
+
38
+ @classmethod
39
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
40
+ return [torch.bfloat16, torch.float16]
41
+
42
+ @classmethod
43
+ def get_min_capability(cls) -> int:
44
+ return 89
45
+
46
+ @classmethod
47
+ def get_config_filenames(cls) -> List[str]:
48
+ return []
49
+
50
+ @classmethod
51
+ def from_config(cls, config: Dict[str, Any]) -> "FBGEMMFp8Config":
52
+ ignore_list = cls.get_from_keys(config, ["modules_to_not_convert"])
53
+ input_scale_ub = cls.get_from_keys(config, ["activation_scale_ub"])
54
+ return cls(ignore_list=ignore_list, input_scale_ub=input_scale_ub)
55
+
56
+ def _is_layer_skipped(self, prefix: str) -> bool:
57
+ # prefix: model.layers.0.self_attn.q_proj
58
+ # proj_name: q_proj
59
+ proj_name = prefix.split(".")[-1]
60
+ if proj_name in _FUSED_LAYER_NAME_MAPPING:
61
+ shard_prefixes = [
62
+ prefix.replace(proj_name, shard_proj_name)
63
+ for shard_proj_name in _FUSED_LAYER_NAME_MAPPING[proj_name]
64
+ ]
65
+
66
+ is_skipped = None
67
+ for shard_prefix in shard_prefixes:
68
+ is_shard_skipped = shard_prefix in self.ignore_list
69
+
70
+ if is_skipped is None:
71
+ is_skipped = is_shard_skipped
72
+ elif is_shard_skipped != is_skipped:
73
+ raise ValueError(
74
+ f"Detected some but not all shards of {prefix} "
75
+ "are quantized. All shards of fused layers "
76
+ "to have the same precision.")
77
+ else:
78
+ is_skipped = prefix in self.ignore_list
79
+
80
+ assert is_skipped is not None
81
+ return is_skipped
82
+
83
+ def get_quant_method(self, layer: torch.nn.Module,
84
+ prefix: str) -> Optional["QuantizeMethodBase"]:
85
+ if isinstance(layer, LinearBase):
86
+ if self._is_layer_skipped(prefix):
87
+ return UnquantizedLinearMethod()
88
+ return FBGEMMFp8LinearMethod(self)
89
+ return None
90
+
91
+ def get_scaled_act_names(self) -> List[str]:
92
+ return []
93
+
94
+
95
+class FBGEMMFp8LinearMethod(LinearMethodBase):
96
+
97
+ def __init__(self, quant_config: FBGEMMFp8Config):
98
+ self.quant_config = quant_config
99
+
100
+ def create_weights(
101
+ self,
102
+ layer: torch.nn.Module,
103
+ input_size_per_partition: int,
104
+ output_partition_sizes: List[int],
105
+ input_size: int,
106
+ output_size: int,
107
+ params_dtype: torch.dtype,
108
+ **extra_weight_attrs,
109
+ ):
110
+ del input_size, output_size
111
+ output_size_per_partition = sum(output_partition_sizes)
112
+
113
+ layer.logical_widths = output_partition_sizes
114
+
115
+ layer.input_size_per_partition = input_size_per_partition
116
+ layer.output_size_per_partition = output_size_per_partition
117
+ layer.orig_dtype = params_dtype
118
+
119
+ # WEIGHT
120
+ weight = Parameter(torch.empty(output_size_per_partition,
121
+ input_size_per_partition,
122
+ dtype=torch.float8_e4m3fn),
123
+ requires_grad=False)
124
+ layer.register_parameter("weight", weight)
125
+ set_weight_attrs(weight, {
126
+ "input_dim": 1,
127
+ "output_dim": 0,
128
+ **extra_weight_attrs,
129
+ })
130
+
131
+ # WEIGHT SCALE
132
+ weight_scale = create_per_channel_scale_param(output_partition_sizes,
133
+ **extra_weight_attrs)
134
+ layer.register_parameter("weight_scale", weight_scale)
135
+
136
+ # INPUT SCALE UPPER BOUND
137
+ input_scale_ub = torch.nn.Parameter(torch.tensor(
138
+ (self.quant_config.input_scale_ub), dtype=torch.float32),
139
+ requires_grad=False)
140
+ layer.input_scale_ub = input_scale_ub
141
+
142
+ def process_weights_after_loading(self, layer: Module) -> None:
143
+ weight = layer.weight
144
+ layer.weight = Parameter(weight.t(), requires_grad=False)
145
+
146
+ def apply(self,
147
+ layer: torch.nn.Module,
148
+ x: torch.Tensor,
149
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
150
+
151
+ return apply_fp8_linear(input=x,
152
+ weight=layer.weight,
153
+ weight_scale=layer.weight_scale,
154
+ input_scale=None,
155
+ input_scale_ub=layer.input_scale_ub,
156
+ bias=bias,
157
+ cutlass_fp8_supported=True,
158
+ use_per_token_if_dynamic=True)
vllm/model_executor/layers/quantization/fp8.py
+2
-2
@@ -66,8 +66,8 @@ class Fp8Config(QuantizationConfig):
66
return cls(is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
67
activation_scheme=activation_scheme)
68
69
- def get_quant_method(
70
- self, layer: torch.nn.Module) -> Optional["QuantizeMethodBase"]:
69
+ def get_quant_method(self, layer: torch.nn.Module,
70
+ prefix: str) -> Optional["QuantizeMethodBase"]:
71
from vllm.attention.layer import Attention # Avoid circular import
72
73
if isinstance(layer, LinearBase):
vllm/model_executor/layers/quantization/gptq.py
+2
-2
@@ -69,8 +69,8 @@ class GPTQConfig(QuantizationConfig):
69
default=False)
70
return cls(weight_bits, group_size, desc_act, lm_head_quantized)
71
72
- def get_quant_method(
73
- self, layer: torch.nn.Module) -> Optional["GPTQLinearMethod"]:
72
+ def get_quant_method(self, layer: torch.nn.Module,
73
+ prefix: str) -> Optional["GPTQLinearMethod"]:
74
if (isinstance(layer, LinearBase) or
75
(isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
76
return GPTQLinearMethod(self)
vllm/model_executor/layers/quantization/gptq_marlin.py
+2
-3
@@ -94,9 +94,8 @@ class GPTQMarlinConfig(QuantizationConfig):
94
" faster inference")
95
return None
96
97
- def get_quant_method(
98
- self,
99
- layer: torch.nn.Module) -> Optional["GPTQMarlinLinearMethod"]:
97
+ def get_quant_method(self, layer: torch.nn.Module,
98
+ prefix: str) -> Optional["GPTQMarlinLinearMethod"]:
99
if (isinstance(layer, LinearBase) or
100
(isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
101
return GPTQMarlinLinearMethod(self)
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+2
-3
@@ -109,9 +109,8 @@ class GPTQMarlin24Config(QuantizationConfig):
109
110
return None
111
112
- def get_quant_method(
113
- self,
114
- layer: torch.nn.Module) -> Optional["GPTQMarlin24LinearMethod"]:
112
+ def get_quant_method(self, layer: torch.nn.Module,
113
+ prefix: str) -> Optional["GPTQMarlin24LinearMethod"]:
114
if isinstance(layer, LinearBase):
115
return GPTQMarlin24LinearMethod(self)
116
return None
vllm/model_executor/layers/quantization/marlin.py
+2
-2
@@ -100,8 +100,8 @@ class MarlinConfig(QuantizationConfig):
100
101
return None
102
103
- def get_quant_method(
104
- self, layer: torch.nn.Module) -> Optional["MarlinLinearMethod"]:
103
+ def get_quant_method(self, layer: torch.nn.Module,
104
+ prefix: str) -> Optional["MarlinLinearMethod"]:
105
if (isinstance(layer, LinearBase) or
106
(isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
107
return MarlinLinearMethod(self)
vllm/model_executor/layers/quantization/squeezellm.py
+2
-2
@@ -52,8 +52,8 @@ class SqueezeLLMConfig(QuantizationConfig):
52
weight_bits = cls.get_from_keys(config, ["wbits"])
53
return cls(weight_bits)
54
55
- def get_quant_method(
56
- self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
55
+ def get_quant_method(self, layer: torch.nn.Module,
56
+ prefix: str) -> Optional[QuantizeMethodBase]:
57
if isinstance(layer, LinearBase):
58
return SqueezeLLMLinearMethod(self)
59
return None
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2
@@ -105,6 +105,7 @@ def apply_fp8_linear(
105
weight: torch.Tensor,
106
weight_scale: torch.Tensor,
107
input_scale: torch.Tensor,
108
+ input_scale_ub: Optional[torch.Tensor] = None,
109
bias: Optional[torch.Tensor] = None,
110
cutlass_fp8_supported: bool = True,
111
use_per_token_if_dynamic: bool = False,
@@ -118,6 +119,7 @@ def apply_fp8_linear(
119
qinput, x_scale = ops.scaled_fp8_quant(
120
input,
121
input_scale,
122
+ scale_ub=input_scale_ub,
123
use_per_token_if_dynamic=use_per_token_if_dynamic)
124
125
# Fused GEMM_DQ
vllm/model_executor/layers/vocab_parallel_embedding.py
+8
-4
@@ -161,6 +161,7 @@ class VocabParallelEmbedding(torch.nn.Module):
161
org_num_embeddings: original vocabulary size (without LoRA).
162
padding_size: padding size for the vocabulary.
163
quant_config: quant config for the layer
164
+ prefix: full name of the layer in the state dict
165
""" # noqa: E501
166
167
def __init__(self,
@@ -169,7 +170,8 @@ class VocabParallelEmbedding(torch.nn.Module):
170
params_dtype: Optional[torch.dtype] = None,
171
org_num_embeddings: Optional[int] = None,
172
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
172
- quant_config: Optional[QuantizationConfig] = None):
173
+ quant_config: Optional[QuantizationConfig] = None,
174
+ prefix: str = ""):
175
super().__init__()
176
177
# Keep the input dimensions.
@@ -195,7 +197,7 @@ class VocabParallelEmbedding(torch.nn.Module):
197
198
linear_method = None
199
if quant_config is not None:
198
- linear_method = quant_config.get_quant_method(self)
200
+ linear_method = quant_config.get_quant_method(self, prefix=prefix)
201
if linear_method is None:
202
linear_method = UnquantizedLinearMethod()
203
self.linear_method: QuantizeMethodBase = linear_method
@@ -382,9 +384,11 @@ class ParallelLMHead(VocabParallelEmbedding):
384
params_dtype: Optional[torch.dtype] = None,
385
org_num_embeddings: Optional[int] = None,
386
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
385
- quant_config: Optional[QuantizationConfig] = None):
387
+ quant_config: Optional[QuantizationConfig] = None,
388
+ prefix: str = ""):
389
super().__init__(num_embeddings, embedding_dim, params_dtype,
387
- org_num_embeddings, padding_size, quant_config)
390
+ org_num_embeddings, padding_size, quant_config,
391
+ prefix)
392
if bias:
393
self.bias = Parameter(
394
torch.empty(self.num_embeddings_per_partition,