@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 683e3cb9c4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Misc ] `fbgemm` checkpoints (#6559)

Robert Shaw committed Jul 20, 2024 at 12:36 UTC 683e3cb9c4b25d10d507817cb0521883d29cc082
24 files changed +234 -47
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
+2 -2
@@ -4,8 +4,8 @@ tasks:
4 - name: "gsm8k"
5 metrics:
6 - name: "exact_match,strict-match"
7 - value: 0.769
7 + value: 0.752
8 - name: "exact_match,flexible-extract"
9 - value: 0.769
9 + value: 0.754
10 limit: 1000
11 num_fewshot: 5
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform -b auto -l 1000 -f 5 -t 1
2 +model_name: "nm-testing/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.753
8 + - name: "exact_match,flexible-extract"
9 + value: 0.753
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1 -1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46 done
47
48 lm_eval --model vllm \
49 - --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray",trust_remote_code=true,max_model_len=4096 \
49 + --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend="ray",trust_remote_code=true,max_model_len=4096 \
50 --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51 --batch_size $BATCH_SIZE
vllm/_custom_ops.py
+2
@@ -315,6 +315,8 @@ def scaled_fp8_quant(
315 Args:
316 input: The input tensor to be quantized to FP8
317 scale: Optional scaling factor for the FP8 quantization
318 + scale_ub: Optional upper bound for scaling factor in dynamic
319 + per token case
320 batch_dim_padding: If specified, pad the first dimension
321 of the output to at least this value.
322 use_per_token_if_dynamic: Whether to do per_tensor or per_token
vllm/attention/layer.py
+2 -1
@@ -34,6 +34,7 @@ class Attention(nn.Module):
34 cache_config: Optional[CacheConfig] = None,
35 quant_config: Optional[QuantizationConfig] = None,
36 blocksparse_params: Optional[Dict[str, Any]] = None,
37 + prefix: str = "",
38 ) -> None:
39 super().__init__()
40 if cache_config is not None:
@@ -56,7 +57,7 @@ class Attention(nn.Module):
57 self._k_scale = 1.0
58 self._v_scale = 1.0
59 quant_method = quant_config.get_quant_method(
59 - self) if quant_config else None
60 + self, prefix=prefix) if quant_config else None
61 if quant_method is not None:
62 assert isinstance(quant_method, Fp8KVCacheMethod)
63 # TODO (mgoin): kv cache dtype should be specified in the FP8
vllm/config.py
+1 -1
@@ -251,7 +251,7 @@ class ModelConfig:
251 f"supported in ROCm.")
252 if (self.quantization
253 not in ("fp8", "marlin", "gptq_marlin_24", "gptq_marlin",
254 - "compressed_tensors")):
254 + "fbgemm_fp8", "compressed_tensors")):
255 logger.warning(
256 "%s quantization is not fully "
257 "optimized yet. The speed can be slower than "
vllm/model_executor/layers/fused_moe/layer.py
+1 -1
@@ -182,7 +182,7 @@ class FusedMoE(torch.nn.Module):
182 self.quant_method: Optional[QuantizeMethodBase] = (
183 UnquantizedFusedMoEMethod())
184 else:
185 - self.quant_method = quant_config.get_quant_method(self)
185 + self.quant_method = quant_config.get_quant_method(self, prefix)
186 assert self.quant_method is not None
187
188 self.quant_method.create_weights(
vllm/model_executor/layers/linear.py
+16 -10
@@ -141,6 +141,7 @@ class LinearBase(torch.nn.Module):
141 skip_bias_add: bool = False,
142 params_dtype: Optional[torch.dtype] = None,
143 quant_config: Optional[QuantizationConfig] = None,
144 + prefix: str = "",
145 ):
146 super().__init__()
147
@@ -155,7 +156,8 @@ class LinearBase(torch.nn.Module):
156 self.quant_method: Optional[
157 QuantizeMethodBase] = UnquantizedLinearMethod()
158 else:
158 - self.quant_method = quant_config.get_quant_method(self)
159 + self.quant_method = quant_config.get_quant_method(self,
160 + prefix=prefix)
161
162 def forward(self, x: torch.Tensor) -> torch.Tensor:
163 raise NotImplementedError
@@ -182,9 +184,13 @@ class ReplicatedLinear(LinearBase):
184 skip_bias_add: bool = False,
185 params_dtype: Optional[torch.dtype] = None,
186 quant_config: Optional[QuantizationConfig] = None,
185 - prefix: Optional[str] = None):
186 - super().__init__(input_size, output_size, skip_bias_add, params_dtype,
187 - quant_config)
187 + prefix: str = ""):
188 + super().__init__(input_size,
189 + output_size,
190 + skip_bias_add,
191 + params_dtype,
192 + quant_config,
193 + prefix=prefix)
194
195 # All the linear layer supports quant method.
196 assert self.quant_method is not None
@@ -258,9 +264,9 @@ class ColumnParallelLinear(LinearBase):
264 params_dtype: Optional[torch.dtype] = None,
265 quant_config: Optional[QuantizationConfig] = None,
266 output_sizes: Optional[List[int]] = None,
261 - prefix: Optional[str] = None):
267 + prefix: str = ""):
268 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
263 - quant_config)
269 + quant_config, prefix)
270
271 self.gather_output = gather_output
272
@@ -370,7 +376,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
376 skip_bias_add: bool = False,
377 params_dtype: Optional[torch.dtype] = None,
378 quant_config: Optional[QuantizationConfig] = None,
373 - prefix: Optional[str] = None):
379 + prefix: str = ""):
380 self.output_sizes = output_sizes
381 tp_size = get_tensor_model_parallel_world_size()
382 assert all(output_size % tp_size == 0 for output_size in output_sizes)
@@ -514,7 +520,7 @@ class QKVParallelLinear(ColumnParallelLinear):
520 skip_bias_add: bool = False,
521 params_dtype: Optional[torch.dtype] = None,
522 quant_config: Optional[QuantizationConfig] = None,
517 - prefix: Optional[str] = None):
523 + prefix: str = ""):
524 self.hidden_size = hidden_size
525 self.head_size = head_size
526 self.total_num_heads = total_num_heads
@@ -707,9 +713,9 @@ class RowParallelLinear(LinearBase):
713 params_dtype: Optional[torch.dtype] = None,
714 reduce_results: bool = True,
715 quant_config: Optional[QuantizationConfig] = None,
710 - prefix: Optional[str] = None):
716 + prefix: str = ""):
717 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
712 - quant_config)
718 + quant_config, prefix)
719
720 self.input_is_parallel = input_is_parallel
721 self.reduce_results = reduce_results
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
10 CompressedTensorsConfig)
11 from vllm.model_executor.layers.quantization.deepspeedfp import (
12 DeepSpeedFPConfig)
13 +from vllm.model_executor.layers.quantization.fbgemm_fp8 import FBGEMMFp8Config
14 from vllm.model_executor.layers.quantization.fp8 import Fp8Config
15 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
16 from vllm.model_executor.layers.quantization.gptq_marlin import (
@@ -24,6 +25,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
25 "awq": AWQConfig,
26 "deepspeedfp": DeepSpeedFPConfig,
27 "fp8": Fp8Config,
28 + "fbgemm_fp8": FBGEMMFp8Config,
29 # The order of gptq methods is important for config.py iteration over
30 # override_quantization_method(..)
31 "marlin": MarlinConfig,
vllm/model_executor/layers/quantization/aqlm.py
+2 -2
@@ -207,8 +207,8 @@ class AQLMConfig(QuantizationConfig):
207 return cls(in_group_size, nbits_per_codebook, num_code_books,
208 out_group_size)
209
210 - def get_quant_method(
211 - self, layer: torch.nn.Module) -> Optional["AQLMLinearMethod"]:
210 + def get_quant_method(self, layer: torch.nn.Module,
211 + prefix: str) -> Optional["AQLMLinearMethod"]:
212 if isinstance(layer, LinearBase):
213 return AQLMLinearMethod(self)
214 return None
vllm/model_executor/layers/quantization/awq.py
+2 -2
@@ -63,8 +63,8 @@ class AWQConfig(QuantizationConfig):
63 zero_point = cls.get_from_keys(config, ["zero_point"])
64 return cls(weight_bits, group_size, zero_point)
65
66 - def get_quant_method(
67 - self, layer: torch.nn.Module) -> Optional["AWQLinearMethod"]:
66 + def get_quant_method(self, layer: torch.nn.Module,
67 + prefix: str) -> Optional["AWQLinearMethod"]:
68 if isinstance(layer, LinearBase):
69 return AWQLinearMethod(self)
70 return None
vllm/model_executor/layers/quantization/base_config.py
+3 -2
@@ -97,12 +97,13 @@ class QuantizationConfig(ABC):
97 return default
98
99 @abstractmethod
100 - def get_quant_method(
101 - self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
100 + def get_quant_method(self, layer: torch.nn.Module,
101 + prefix: str) -> Optional[QuantizeMethodBase]:
102 """Get the quantize method to use for the quantized layer.
103
104 Args:
105 layer: The layer for the quant method.
106 + prefix: The full name of the layer in the state dict
107 Returns:
108 The quantize method. None if the given layer doesn't support quant
109 method.
vllm/model_executor/layers/quantization/bitsandbytes.py
+2 -3
@@ -60,9 +60,8 @@ class BitsAndBytesConfig(QuantizationConfig):
60 target_modules = cls.get_from_keys(config, ["target_modules"])
61 return cls(adapter_name, target_modules)
62
63 - def get_quant_method(
64 - self,
65 - layer: torch.nn.Module) -> Optional["BitsAndBytesLinearMethod"]:
63 + def get_quant_method(self, layer: torch.nn.Module,
64 + prefix: str) -> Optional["BitsAndBytesLinearMethod"]:
65 if isinstance(layer, LinearBase):
66 return BitsAndBytesLinearMethod(self)
67 return None
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5 -1
@@ -44,8 +44,12 @@ class CompressedTensorsConfig(QuantizationConfig):
44 def get_name(self) -> str:
45 return "compressed_tensors"
46
47 + # TODO (@robertgshaw2-neuralmagic): do layer skipping though here
48 + # rather than though create_weights to match other methods
49 def get_quant_method(
48 - self, layer: torch.nn.Module
50 + self,
51 + layer: torch.nn.Module,
52 + prefix: str,
53 ) -> Optional["CompressedTensorsLinearMethod"]:
54 if isinstance(layer, LinearBase):
55 return CompressedTensorsLinearMethod(self)
vllm/model_executor/layers/quantization/deepspeedfp.py
+2 -3
@@ -69,9 +69,8 @@ class DeepSpeedFPConfig(QuantizationConfig):
69 "quantize_config.json",
70 ]
71
72 - def get_quant_method(
73 - self,
74 - layer: torch.nn.Module) -> Optional["DeepSpeedFPLinearMethod"]:
72 + def get_quant_method(self, layer: torch.nn.Module,
73 + prefix: str) -> Optional["DeepSpeedFPLinearMethod"]:
74 if isinstance(layer, LinearBase):
75 return DeepSpeedFPLinearMethod(self)
76 return None
vllm/model_executor/layers/quantization/fbgemm_fp8.py new
+158
@@ -0,0 +1,158 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +from torch.nn import Module
5 +from torch.nn.parameter import Parameter
6 +
7 +from vllm.logger import init_logger
8 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9 + UnquantizedLinearMethod)
10 +from vllm.model_executor.layers.quantization.base_config import (
11 + QuantizationConfig, QuantizeMethodBase)
12 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
13 + apply_fp8_linear, create_per_channel_scale_param)
14 +from vllm.model_executor.utils import set_weight_attrs
15 +
16 +logger = init_logger(__name__)
17 +
18 +# Note: this is a hack. We should update each model to register the
19 +# stacked params and get it from there instead in a future PR.
20 +# fused_name: List[shard_name]
21 +_FUSED_LAYER_NAME_MAPPING = {
22 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
23 + "gate_up_proj": ["gate_proj", "up_proj"]
24 +}
25 +
26 +
27 +class FBGEMMFp8Config(QuantizationConfig):
28 + """Config class for FBGEMM Fp8."""
29 +
30 + def __init__(self, ignore_list: List[str], input_scale_ub: float):
31 + self.ignore_list = ignore_list
32 + self.input_scale_ub = input_scale_ub
33 +
34 + @classmethod
35 + def get_name(cls) -> str:
36 + return "fbgemm_fp8"
37 +
38 + @classmethod
39 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
40 + return [torch.bfloat16, torch.float16]
41 +
42 + @classmethod
43 + def get_min_capability(cls) -> int:
44 + return 89
45 +
46 + @classmethod
47 + def get_config_filenames(cls) -> List[str]:
48 + return []
49 +
50 + @classmethod
51 + def from_config(cls, config: Dict[str, Any]) -> "FBGEMMFp8Config":
52 + ignore_list = cls.get_from_keys(config, ["modules_to_not_convert"])
53 + input_scale_ub = cls.get_from_keys(config, ["activation_scale_ub"])
54 + return cls(ignore_list=ignore_list, input_scale_ub=input_scale_ub)
55 +
56 + def _is_layer_skipped(self, prefix: str) -> bool:
57 + # prefix: model.layers.0.self_attn.q_proj
58 + # proj_name: q_proj
59 + proj_name = prefix.split(".")[-1]
60 + if proj_name in _FUSED_LAYER_NAME_MAPPING:
61 + shard_prefixes = [
62 + prefix.replace(proj_name, shard_proj_name)
63 + for shard_proj_name in _FUSED_LAYER_NAME_MAPPING[proj_name]
64 + ]
65 +
66 + is_skipped = None
67 + for shard_prefix in shard_prefixes:
68 + is_shard_skipped = shard_prefix in self.ignore_list
69 +
70 + if is_skipped is None:
71 + is_skipped = is_shard_skipped
72 + elif is_shard_skipped != is_skipped:
73 + raise ValueError(
74 + f"Detected some but not all shards of {prefix} "
75 + "are quantized. All shards of fused layers "
76 + "to have the same precision.")
77 + else:
78 + is_skipped = prefix in self.ignore_list
79 +
80 + assert is_skipped is not None
81 + return is_skipped
82 +
83 + def get_quant_method(self, layer: torch.nn.Module,
84 + prefix: str) -> Optional["QuantizeMethodBase"]:
85 + if isinstance(layer, LinearBase):
86 + if self._is_layer_skipped(prefix):
87 + return UnquantizedLinearMethod()
88 + return FBGEMMFp8LinearMethod(self)
89 + return None
90 +
91 + def get_scaled_act_names(self) -> List[str]:
92 + return []
93 +
94 +
95 +class FBGEMMFp8LinearMethod(LinearMethodBase):
96 +
97 + def __init__(self, quant_config: FBGEMMFp8Config):
98 + self.quant_config = quant_config
99 +
100 + def create_weights(
101 + self,
102 + layer: torch.nn.Module,
103 + input_size_per_partition: int,
104 + output_partition_sizes: List[int],
105 + input_size: int,
106 + output_size: int,
107 + params_dtype: torch.dtype,
108 + **extra_weight_attrs,
109 + ):
110 + del input_size, output_size
111 + output_size_per_partition = sum(output_partition_sizes)
112 +
113 + layer.logical_widths = output_partition_sizes
114 +
115 + layer.input_size_per_partition = input_size_per_partition
116 + layer.output_size_per_partition = output_size_per_partition
117 + layer.orig_dtype = params_dtype
118 +
119 + # WEIGHT
120 + weight = Parameter(torch.empty(output_size_per_partition,
121 + input_size_per_partition,
122 + dtype=torch.float8_e4m3fn),
123 + requires_grad=False)
124 + layer.register_parameter("weight", weight)
125 + set_weight_attrs(weight, {
126 + "input_dim": 1,
127 + "output_dim": 0,
128 + **extra_weight_attrs,
129 + })
130 +
131 + # WEIGHT SCALE
132 + weight_scale = create_per_channel_scale_param(output_partition_sizes,
133 + **extra_weight_attrs)
134 + layer.register_parameter("weight_scale", weight_scale)
135 +
136 + # INPUT SCALE UPPER BOUND
137 + input_scale_ub = torch.nn.Parameter(torch.tensor(
138 + (self.quant_config.input_scale_ub), dtype=torch.float32),
139 + requires_grad=False)
140 + layer.input_scale_ub = input_scale_ub
141 +
142 + def process_weights_after_loading(self, layer: Module) -> None:
143 + weight = layer.weight
144 + layer.weight = Parameter(weight.t(), requires_grad=False)
145 +
146 + def apply(self,
147 + layer: torch.nn.Module,
148 + x: torch.Tensor,
149 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
150 +
151 + return apply_fp8_linear(input=x,
152 + weight=layer.weight,
153 + weight_scale=layer.weight_scale,
154 + input_scale=None,
155 + input_scale_ub=layer.input_scale_ub,
156 + bias=bias,
157 + cutlass_fp8_supported=True,
158 + use_per_token_if_dynamic=True)
vllm/model_executor/layers/quantization/fp8.py
+2 -2
@@ -66,8 +66,8 @@ class Fp8Config(QuantizationConfig):
66 return cls(is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
67 activation_scheme=activation_scheme)
68
69 - def get_quant_method(
70 - self, layer: torch.nn.Module) -> Optional["QuantizeMethodBase"]:
69 + def get_quant_method(self, layer: torch.nn.Module,
70 + prefix: str) -> Optional["QuantizeMethodBase"]:
71 from vllm.attention.layer import Attention # Avoid circular import
72
73 if isinstance(layer, LinearBase):
vllm/model_executor/layers/quantization/gptq.py
+2 -2
@@ -69,8 +69,8 @@ class GPTQConfig(QuantizationConfig):
69 default=False)
70 return cls(weight_bits, group_size, desc_act, lm_head_quantized)
71
72 - def get_quant_method(
73 - self, layer: torch.nn.Module) -> Optional["GPTQLinearMethod"]:
72 + def get_quant_method(self, layer: torch.nn.Module,
73 + prefix: str) -> Optional["GPTQLinearMethod"]:
74 if (isinstance(layer, LinearBase) or
75 (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
76 return GPTQLinearMethod(self)
vllm/model_executor/layers/quantization/gptq_marlin.py
+2 -3
@@ -94,9 +94,8 @@ class GPTQMarlinConfig(QuantizationConfig):
94 " faster inference")
95 return None
96
97 - def get_quant_method(
98 - self,
99 - layer: torch.nn.Module) -> Optional["GPTQMarlinLinearMethod"]:
97 + def get_quant_method(self, layer: torch.nn.Module,
98 + prefix: str) -> Optional["GPTQMarlinLinearMethod"]:
99 if (isinstance(layer, LinearBase) or
100 (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
101 return GPTQMarlinLinearMethod(self)
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+2 -3
@@ -109,9 +109,8 @@ class GPTQMarlin24Config(QuantizationConfig):
109
110 return None
111
112 - def get_quant_method(
113 - self,
114 - layer: torch.nn.Module) -> Optional["GPTQMarlin24LinearMethod"]:
112 + def get_quant_method(self, layer: torch.nn.Module,
113 + prefix: str) -> Optional["GPTQMarlin24LinearMethod"]:
114 if isinstance(layer, LinearBase):
115 return GPTQMarlin24LinearMethod(self)
116 return None
vllm/model_executor/layers/quantization/marlin.py
+2 -2
@@ -100,8 +100,8 @@ class MarlinConfig(QuantizationConfig):
100
101 return None
102
103 - def get_quant_method(
104 - self, layer: torch.nn.Module) -> Optional["MarlinLinearMethod"]:
103 + def get_quant_method(self, layer: torch.nn.Module,
104 + prefix: str) -> Optional["MarlinLinearMethod"]:
105 if (isinstance(layer, LinearBase) or
106 (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
107 return MarlinLinearMethod(self)
vllm/model_executor/layers/quantization/squeezellm.py
+2 -2
@@ -52,8 +52,8 @@ class SqueezeLLMConfig(QuantizationConfig):
52 weight_bits = cls.get_from_keys(config, ["wbits"])
53 return cls(weight_bits)
54
55 - def get_quant_method(
56 - self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
55 + def get_quant_method(self, layer: torch.nn.Module,
56 + prefix: str) -> Optional[QuantizeMethodBase]:
57 if isinstance(layer, LinearBase):
58 return SqueezeLLMLinearMethod(self)
59 return None
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2
@@ -105,6 +105,7 @@ def apply_fp8_linear(
105 weight: torch.Tensor,
106 weight_scale: torch.Tensor,
107 input_scale: torch.Tensor,
108 + input_scale_ub: Optional[torch.Tensor] = None,
109 bias: Optional[torch.Tensor] = None,
110 cutlass_fp8_supported: bool = True,
111 use_per_token_if_dynamic: bool = False,
@@ -118,6 +119,7 @@ def apply_fp8_linear(
119 qinput, x_scale = ops.scaled_fp8_quant(
120 input,
121 input_scale,
122 + scale_ub=input_scale_ub,
123 use_per_token_if_dynamic=use_per_token_if_dynamic)
124
125 # Fused GEMM_DQ
vllm/model_executor/layers/vocab_parallel_embedding.py
+8 -4
@@ -161,6 +161,7 @@ class VocabParallelEmbedding(torch.nn.Module):
161 org_num_embeddings: original vocabulary size (without LoRA).
162 padding_size: padding size for the vocabulary.
163 quant_config: quant config for the layer
164 + prefix: full name of the layer in the state dict
165 """ # noqa: E501
166
167 def __init__(self,
@@ -169,7 +170,8 @@ class VocabParallelEmbedding(torch.nn.Module):
170 params_dtype: Optional[torch.dtype] = None,
171 org_num_embeddings: Optional[int] = None,
172 padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
172 - quant_config: Optional[QuantizationConfig] = None):
173 + quant_config: Optional[QuantizationConfig] = None,
174 + prefix: str = ""):
175 super().__init__()
176
177 # Keep the input dimensions.
@@ -195,7 +197,7 @@ class VocabParallelEmbedding(torch.nn.Module):
197
198 linear_method = None
199 if quant_config is not None:
198 - linear_method = quant_config.get_quant_method(self)
200 + linear_method = quant_config.get_quant_method(self, prefix=prefix)
201 if linear_method is None:
202 linear_method = UnquantizedLinearMethod()
203 self.linear_method: QuantizeMethodBase = linear_method
@@ -382,9 +384,11 @@ class ParallelLMHead(VocabParallelEmbedding):
384 params_dtype: Optional[torch.dtype] = None,
385 org_num_embeddings: Optional[int] = None,
386 padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
385 - quant_config: Optional[QuantizationConfig] = None):
387 + quant_config: Optional[QuantizationConfig] = None,
388 + prefix: str = ""):
389 super().__init__(num_embeddings, embedding_dim, params_dtype,
387 - org_num_embeddings, padding_size, quant_config)
390 + org_num_embeddings, padding_size, quant_config,
391 + prefix)
392 if bias:
393 self.bias = Parameter(
394 torch.empty(self.num_embeddings_per_partition,