A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[CORE] [QUANT] Support for GPTQModel's `dynamic` quantization per module override/control (#7086)
Qubitium-ModelCloud committed
Feb 13, 2025 at 01:19 UTC
36a08630e80b6176489eef45e15b07724d95a944
8 files changed
+281
-56
tests/quantization/test_gptq_dynamic.py
new
+68
@@ -0,0 +1,68 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""Tests whether gptq models with dynamic quantized can be loaded.
3
+
4
+Run `pytest tests/quantization/test_gptq_dynamic.py --forked`.
5
+"""
6
+
7
+import pytest
8
+import torch
9
+
10
+from vllm.model_executor.layers.linear import UnquantizedLinearMethod
11
+from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
12
+from vllm.model_executor.layers.quantization.gptq_marlin import (
13
+ GPTQMarlinLinearMethod)
14
+from vllm.model_executor.layers.quantization.utils.gptq_utils import (
15
+ get_dynamic_override)
16
+
17
+PROMPT = "On the surface of Mars, we found"
18
+
19
+# The first layer is quantized using bits=4, group_size=128
20
+# The second layer is quantized using bits=8, group_size=32
21
+# All other layers (layer index >= 2) are not quantized
22
+MODEL_QUANT = [
23
+ ("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
24
+ True),
25
+ ("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse",
26
+ False),
27
+]
28
+
29
+
30
+@pytest.mark.parametrize("model_id, use_marlin_kernel", MODEL_QUANT)
31
+def test_gptq_with_dynamic(vllm_runner, model_id: str,
32
+ use_marlin_kernel: bool):
33
+
34
+ vllm_model = vllm_runner(model_id, dtype=torch.float16, max_model_len=2048)
35
+
36
+ linear_method_cls = GPTQMarlinLinearMethod if use_marlin_kernel else (
37
+ GPTQLinearMethod)
38
+
39
+ for name, submodule in (vllm_model.model.llm_engine.model_executor.
40
+ driver_worker.model_runner.model.named_modules()):
41
+ if name == "lm_head":
42
+ assert isinstance(submodule.quant_method, linear_method_cls)
43
+ elif name == 'model.layers.0.self_attn.qkv_proj':
44
+ # The first layer is quantized using bits=4, group_size=128
45
+ # desc_act=True
46
+ assert isinstance(submodule.quant_method, linear_method_cls)
47
+ config = submodule.quant_method.quant_config
48
+ assert config.weight_bits == 4
49
+ assert config.group_size == 128
50
+ assert config.desc_act
51
+ elif name == 'model.layers.1.self_attn.qkv_proj':
52
+ # The second layer is quantized using bits=8, group_size=32
53
+ # desc_act=False
54
+ assert isinstance(submodule.quant_method, linear_method_cls)
55
+ config = submodule.quant_method.quant_config
56
+ assert get_dynamic_override(config, layer_name=name,
57
+ key="bits") == 8
58
+ assert get_dynamic_override(config,
59
+ layer_name=name,
60
+ key="group_size") == 32
61
+ assert not get_dynamic_override(
62
+ config, layer_name=name, key="desc_act")
63
+ elif (name == 'model.layers.2.self_attn.qkv_proj'
64
+ or name == 'model.layers.2.mlp.gate_up_proj'):
65
+ # All other layers (layer index >= 2) are not quantized
66
+ assert isinstance(submodule.quant_method, UnquantizedLinearMethod)
67
+
68
+ del vllm_model
tests/quantization/test_lm_head.py
+12
-13
@@ -3,7 +3,6 @@
3
4
Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`.
5
"""
6
-from typing import Tuple
6
7
import pytest
8
import torch
@@ -17,31 +16,31 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
16
17
PROMPT = "On the surface of Mars, we found"
18
20
-MODELS_QUANT = [(
21
- "LnL-AI/TinyLlama-1.1B-intermediate-step-1341k-3T-autoround-lm_head-symFalse",
22
- True), ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
23
- ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", False)]
19
+MODELS_QUANT = [
20
+ ("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head", True),
21
+ ("ModelCloud/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit-10-25-2024", False),
22
+ ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
23
+ ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", False)
24
+]
25
26
26
-@pytest.mark.parametrize("model_lm_head_quant", MODELS_QUANT)
27
+@pytest.mark.parametrize("model_id, lm_head_quantized", MODELS_QUANT)
28
def test_lm_head(
29
vllm_runner,
29
- model_lm_head_quant: Tuple[str, bool],
30
+ model_id: str,
31
+ lm_head_quantized: bool,
32
) -> None:
31
- model, lm_head_quantized = model_lm_head_quant
32
-
33
- with vllm_runner(model, dtype=torch.float16,
33
+ with vllm_runner(model_id, dtype=torch.float16,
34
max_model_len=2048) as vllm_model:
35
36
def check_model(model):
37
lm_head_layer = model.lm_head
38
-
38
if lm_head_quantized:
40
- assert isinstance(lm_head_layer.linear_method,
39
+ assert isinstance(lm_head_layer.quant_method,
40
(GPTQLinearMethod, GPTQMarlinLinearMethod,
41
MarlinLinearMethod))
42
else:
44
- assert isinstance(lm_head_layer.linear_method,
43
+ assert isinstance(lm_head_layer.quant_method,
44
UnquantizedEmbeddingMethod)
45
46
vllm_model.apply_model(check_model)
vllm/lora/layers.py
+1
-1
@@ -1039,7 +1039,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1039
embedding_bias: Optional[torch.Tensor] = None,
1040
) -> Optional[torch.Tensor]:
1041
# Get the logits for the next tokens.
1042
- logits = lm_head.linear_method.apply(lm_head, hidden_states)
1042
+ logits = lm_head.quant_method.apply(lm_head, hidden_states)
1043
if embedding_bias is not None:
1044
logits += embedding_bias
1045
vllm/model_executor/layers/logits_processor.py
+3
-3
@@ -108,9 +108,9 @@ class LogitsProcessor(nn.Module):
108
embedding_bias: Optional[torch.Tensor],
109
) -> Optional[torch.Tensor]:
110
# Get the logits for the next tokens.
111
- logits = lm_head.linear_method.apply(lm_head,
112
- hidden_states,
113
- bias=embedding_bias)
111
+ logits = lm_head.quant_method.apply(lm_head,
112
+ hidden_states,
113
+ bias=embedding_bias)
114
115
# Gather logits for TP
116
logits = self._gather_logits(logits)
vllm/model_executor/layers/quantization/gptq.py
+38
-9
@@ -3,16 +3,17 @@
3
import enum
4
from enum import Enum
5
from fractions import Fraction
6
-from typing import Any, Dict, List, Optional
6
+from typing import Any, Dict, List, Optional, Union
7
8
import torch
9
from torch.nn.parameter import Parameter
10
11
from vllm import _custom_ops as ops
12
-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12
+from vllm.model_executor.layers.linear import LinearMethodBase
13
from vllm.model_executor.layers.quantization.base_config import (
14
QuantizationConfig)
15
-from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15
+from vllm.model_executor.layers.quantization.utils.gptq_utils import (
16
+ get_linear_quant_method)
17
from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
18
GroupQuantScaleParameter,
19
PackedColumnParameter,
@@ -32,7 +33,33 @@ class GPTQConfig(QuantizationConfig):
33
group_size: int,
34
desc_act: bool,
35
lm_head_quantized: bool,
36
+ dynamic: Dict[str, Dict[str, Union[int, bool]]],
37
) -> None:
38
+ # GPTQModel use `dynamic` config property to allow per module
39
+ # quantization config so each module can be individually optimized.
40
+ # Format is Dict[str, Dict] where key is a regex string that can
41
+ # perform both positive ("+:" prefixed) or negative ("-:" prefixed)
42
+ # matching of a module.
43
+ # Default to positive match, override base quant config mode, if no
44
+ # prefix is used. Value is in dict format of field key and override
45
+ # value.
46
+ # Negative matching will skip quantization init for this module
47
+ # entirely:
48
+ # non-quantized inference. More details and quantization examples can be
49
+ # found at: https://github.com/ModelCloud/GPTQModel
50
+ # Example:
51
+ # # last 1/2 of the layers 10-21 has 8bit vs 4bit for 0-9
52
+ # # last 1/4 of the layers 16-21 has 8bit and group_size 64
53
+ # dynamic = {
54
+ # #`.*\.` matches the layers_node prefix
55
+ # # positive match layer 10-15
56
+ # r"+:.*\.(?:1[0-5])\..*": {"bits": 8,},
57
+ # # positive match layer 16-21
58
+ # r"+:.*\.(?:1[6-9]|20|21)\..*": {"bits": 8, "group_size": 64,},
59
+ # r"-:.*\.moe\..*": {}, # negative match (skip) all `moe` layers
60
+ # }
61
+ self.dynamic = dynamic
62
+
63
self.weight_bits = weight_bits
64
self.group_size = group_size
65
self.desc_act = desc_act
@@ -47,7 +74,8 @@ class GPTQConfig(QuantizationConfig):
74
return (f"GPTQConfig(weight_bits={self.weight_bits}, "
75
f"group_size={self.group_size}, "
76
f"desc_act={self.desc_act}),"
50
- f"lm_head_quantized={self.lm_head_quantized}")
77
+ f"lm_head_quantized={self.lm_head_quantized}), "
78
+ f"dynamic={self.dynamic}")
79
80
@classmethod
81
def get_name(cls) -> str:
@@ -68,19 +96,20 @@ class GPTQConfig(QuantizationConfig):
96
97
@classmethod
98
def from_config(cls, config: Dict[str, Any]) -> "GPTQConfig":
99
+ dynamic = cls.get_from_keys_or(config, ["dynamic"], default={})
100
+ dynamic = {} if dynamic is None else dynamic
101
+
102
weight_bits = cls.get_from_keys(config, ["bits"])
103
group_size = cls.get_from_keys(config, ["group_size"])
104
desc_act = cls.get_from_keys(config, ["desc_act"])
105
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
106
default=False)
76
- return cls(weight_bits, group_size, desc_act, lm_head_quantized)
107
+ return cls(weight_bits, group_size, desc_act, lm_head_quantized,
108
+ dynamic)
109
110
def get_quant_method(self, layer: torch.nn.Module,
111
prefix: str) -> Optional["GPTQLinearMethod"]:
80
- if (isinstance(layer, LinearBase) or
81
- (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
82
- return GPTQLinearMethod(self)
83
- return None
112
+ return get_linear_quant_method(self, layer, prefix, GPTQLinearMethod)
113
114
115
class ExllamaState(Enum):
vllm/model_executor/layers/quantization/gptq_marlin.py
+47
-12
@@ -9,17 +9,21 @@ from vllm import _custom_ops as ops
9
from vllm.logger import init_logger
10
from vllm.model_executor.layers.fused_moe.layer import (
11
FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
12
-from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
12
+from vllm.model_executor.layers.linear import (LinearMethodBase,
13
+ UnquantizedLinearMethod,
14
set_weight_attrs)
15
from vllm.model_executor.layers.quantization.base_config import (
16
QuantizationConfig)
17
from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
18
MPLinearLayerConfig, choose_mp_linear_kernel)
19
from vllm.model_executor.layers.quantization.utils import replace_parameter
20
+from vllm.model_executor.layers.quantization.utils.gptq_utils import (
21
+ get_linear_quant_method)
22
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
23
check_marlin_supported, marlin_moe_permute_scales,
24
marlin_repeat_scales_on_all_ranks, verify_marlin_supported)
22
-from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
25
+from vllm.model_executor.layers.vocab_parallel_embedding import (
26
+ UnquantizedEmbeddingMethod)
27
from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
28
GroupQuantScaleParameter,
29
PackedColumnParameter,
@@ -47,12 +51,41 @@ class GPTQMarlinConfig(QuantizationConfig):
51
desc_act: bool,
52
is_sym: bool,
53
lm_head_quantized: bool,
54
+ dynamic: Dict[str, Dict[str, Union[int, bool]]],
55
) -> None:
56
if desc_act and group_size == -1:
57
# In this case, act_order == True is the same as act_order == False
58
# (since we have only one group per output channel)
59
desc_act = False
60
61
+ # GPTQModel use `dynamic` config property to allow per module
62
+ # quantization config so each module can be individually optimized.
63
+ # Format is Dict[str, Dict] where key is a regex string that can
64
+ # perform both positive ("+:" prefixed) or negative ("-:" prefixed)
65
+ # matching of a module.
66
+ # Default to positive match, override base quant config mode, if no
67
+ # prefix is used. Value is in dict format of field key and override
68
+ # value.
69
+ # Negative matching will skip quantization init for this module
70
+ # entirely:
71
+ # non-quantized inference. More details and quantization examples can be
72
+ # found at: https://github.com/ModelCloud/GPTQModel
73
+ # Example:
74
+ # # last 1/2 of the layers 10-21 has 8bit vs 4bit for 0-9
75
+ # # last 1/4 of the layers 16-21 has 8bit and group_size 64
76
+ # dynamic = {
77
+ # #`.*\.` matches the layers_node prefix
78
+ # # positive match layer 10-15
79
+ # r"+:.*\.(?:1[0-5])\..*": {"bits": 8,},
80
+ # # positive match layer 16-21
81
+ # r"+:.*\.(?:1[6-9]|20|21)\..*": {"bits": 8, "group_size": 64,},
82
+ # r"-:.*\.moe\..*": {}, # negative match (skip) all `moe` layers
83
+ # }
84
+ self.dynamic = dynamic
85
+
86
+ self.weight_bits = weight_bits
87
+ self.is_sym = is_sym
88
+
89
self.pack_factor = 32 // weight_bits # packed into int32
90
self.group_size = group_size
91
self.desc_act = desc_act
@@ -68,7 +101,8 @@ class GPTQMarlinConfig(QuantizationConfig):
101
return (f"GPTQMarlinConfig(quant_type={self.quant_type}, "
102
f"group_size={self.group_size}, "
103
f"desc_act={self.desc_act}, "
71
- f"lm_head_quantized={self.lm_head_quantized})")
104
+ f"lm_head_quantized={self.lm_head_quantized}), "
105
+ f"dynamic={self.dynamic}")
106
107
@classmethod
108
def get_name(cls) -> str:
@@ -88,6 +122,9 @@ class GPTQMarlinConfig(QuantizationConfig):
122
123
@classmethod
124
def from_config(cls, config: Dict[str, Any]) -> "GPTQMarlinConfig":
125
+ dynamic = cls.get_from_keys_or(config, ["dynamic"], default={})
126
+ dynamic = {} if dynamic is None else dynamic
127
+
128
weight_bits = cls.get_from_keys(config, ["bits"])
129
group_size = cls.get_from_keys(config, ["group_size"])
130
desc_act = cls.get_from_keys(config, ["desc_act"])
@@ -95,7 +132,7 @@ class GPTQMarlinConfig(QuantizationConfig):
132
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
133
default=False)
134
return cls(weight_bits, group_size, desc_act, is_sym,
98
- lm_head_quantized)
135
+ lm_head_quantized, dynamic)
136
137
@classmethod
138
def override_quantization_method(cls, hf_quant_cfg,
@@ -120,17 +157,15 @@ class GPTQMarlinConfig(QuantizationConfig):
157
158
def get_quant_method(
159
self, layer: torch.nn.Module, prefix: str
123
- ) -> Optional[Union["GPTQMarlinLinearMethod", "GPTQMarlinMoEMethod"]]:
124
- if isinstance(layer, LinearBase) or (isinstance(layer, ParallelLMHead)
125
- and self.lm_head_quantized):
126
- return GPTQMarlinLinearMethod(self)
127
- elif isinstance(layer, FusedMoE):
160
+ ) -> Optional[Union["GPTQMarlinLinearMethod", "GPTQMarlinMoEMethod",
161
+ UnquantizedLinearMethod, UnquantizedEmbeddingMethod]]:
162
+ if isinstance(layer, FusedMoE):
163
return GPTQMarlinMoEMethod(self)
129
- return None
164
+ return get_linear_quant_method(self, layer, prefix,
165
+ GPTQMarlinLinearMethod)
166
167
@classmethod
168
def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]):
133
- # Extract data from quant config.
169
quant_method = quant_config.get("quant_method", "").lower()
170
num_bits = quant_config.get("bits")
171
group_size = quant_config.get("group_size")
@@ -143,7 +178,7 @@ class GPTQMarlinConfig(QuantizationConfig):
178
if quant_method != "gptq":
179
return False
180
146
- # If we cannot find the info needed in the config, cannot convert.
181
+ # Marlin conversion is only valid if required properties are found
182
if (num_bits is None or group_size is None or sym is None
183
or desc_act is None):
184
return False
vllm/model_executor/layers/quantization/utils/gptq_utils.py
new
+94
@@ -0,0 +1,94 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+import re
3
+from copy import deepcopy
4
+from typing import Dict, Optional, Union
5
+
6
+import torch
7
+
8
+from vllm.config import QuantizationConfig
9
+from vllm.model_executor.layers.linear import (LinearBase,
10
+ UnquantizedLinearMethod)
11
+from vllm.model_executor.layers.vocab_parallel_embedding import (
12
+ ParallelLMHead, UnquantizedEmbeddingMethod)
13
+
14
+
15
+# Match dynamic rules with module name (prefix) and override quantize
16
+# config if module (prefix) matches a rule
17
+def override_config(config: QuantizationConfig, prefix: str):
18
+ weight_bits = get_dynamic_override(config, prefix, "bits",
19
+ config.weight_bits)
20
+ if isinstance(weight_bits, int):
21
+ config.weight_bits = weight_bits
22
+ group_size = get_dynamic_override(config, prefix, "group_size",
23
+ config.group_size)
24
+ if isinstance(group_size, int):
25
+ config.group_size = group_size
26
+ desc_act = get_dynamic_override(config, prefix, "desc_act",
27
+ config.desc_act)
28
+ if isinstance(desc_act, bool):
29
+ config.desc_act = desc_act
30
+
31
+ config.pack_factor = 32 // config.weight_bits # packed into int32
32
+ if config.get_name() == "gptq_marlin":
33
+ is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
34
+ if isinstance(is_sym, bool):
35
+ config.is_sym = is_sym
36
+
37
+ if (config.weight_bits, config.is_sym) not in config.TYPE_MAP:
38
+ raise ValueError("Unsupported quantization config: "
39
+ f"bits={config.weight_bits}, sym={config.is_sym}")
40
+
41
+ config.quant_type = config.TYPE_MAP[(config.weight_bits,
42
+ config.is_sym)]
43
+ elif config.get_name() == "gptq":
44
+ if config.weight_bits not in [2, 3, 4, 8]:
45
+ raise ValueError(
46
+ "Currently, only 2/3/4/8-bit weight quantization is "
47
+ f"supported for GPTQ, but got {config.weight_bits} bits.")
48
+
49
+
50
+def get_dynamic_override(
51
+ config: QuantizationConfig,
52
+ layer_name: str,
53
+ key: Optional[str] = None,
54
+ default_value: Union[int, bool,
55
+ None] = None) -> Union[Dict, int, bool, None]:
56
+ for pattern, pattern_dict in config.dynamic.items():
57
+ # Negative match: matched modules are excluded from quantized init
58
+ if pattern.startswith("-:"):
59
+ if re.match(pattern.removeprefix("-:"), layer_name):
60
+ return False
61
+ # Positive match: matched modules have quant properties overrides
62
+ # base quant config
63
+ elif re.match(pattern.removeprefix("+:"), layer_name):
64
+ if key is None:
65
+ return pattern_dict
66
+ else:
67
+ return pattern_dict.get(key, default_value)
68
+ return default_value
69
+
70
+
71
+def get_linear_quant_method(
72
+ config: QuantizationConfig,
73
+ layer: torch.nn.Module,
74
+ prefix: str,
75
+ linear_method_cls: type,
76
+):
77
+ cloned_config = deepcopy(config)
78
+ parallel_lm_head_quantized = isinstance(
79
+ layer, ParallelLMHead) and cloned_config.lm_head_quantized
80
+ if isinstance(layer, LinearBase) or parallel_lm_head_quantized:
81
+ # False = skip module, None = no override, else = Positive match
82
+ if get_dynamic_override( # noqa: E712
83
+ cloned_config, # noqa: E712
84
+ layer_name=prefix) == False: # noqa: E712
85
+ if parallel_lm_head_quantized:
86
+ return UnquantizedEmbeddingMethod()
87
+ return UnquantizedLinearMethod()
88
+
89
+ if prefix:
90
+ # Dynamic per module/layer rules may override base config
91
+ override_config(cloned_config, prefix=prefix)
92
+
93
+ return linear_method_cls(cloned_config)
94
+ return None
vllm/model_executor/layers/vocab_parallel_embedding.py
+18
-18
@@ -226,24 +226,24 @@ class VocabParallelEmbedding(torch.nn.Module):
226
self.tp_size)
227
self.embedding_dim = embedding_dim
228
229
- linear_method = None
229
+ quant_method = None
230
if quant_config is not None:
231
- linear_method = quant_config.get_quant_method(self, prefix=prefix)
232
- if linear_method is None:
233
- linear_method = UnquantizedEmbeddingMethod()
231
+ quant_method = quant_config.get_quant_method(self, prefix=prefix)
232
+ if quant_method is None:
233
+ quant_method = UnquantizedEmbeddingMethod()
234
235
# If we are making an embedding layer, then our quantization linear
236
# method must implement the embedding operation. If we are another
237
# layer type like ParallelLMHead, this is not important.
238
is_embedding_layer = type(self.__class__) is VocabParallelEmbedding
239
- linear_method_implements_embedding = method_has_implemented_embedding(
240
- type(linear_method))
241
- if is_embedding_layer and not linear_method_implements_embedding:
239
+ quant_method_implements_embedding = method_has_implemented_embedding(
240
+ type(quant_method))
241
+ if is_embedding_layer and not quant_method_implements_embedding:
242
raise NotImplementedError(
243
- f"The class {type(linear_method).__name__} must implement "
243
+ f"The class {type(quant_method).__name__} must implement "
244
"the 'embedding' method, see UnquantizedEmbeddingMethod.")
245
246
- self.linear_method: QuantizeMethodBase = linear_method
246
+ self.quant_method: QuantizeMethodBase = quant_method
247
248
if params_dtype is None:
249
params_dtype = torch.get_default_dtype()
@@ -260,13 +260,13 @@ class VocabParallelEmbedding(torch.nn.Module):
260
self.shard_indices.added_vocab_end_index -
261
self.shard_indices.added_vocab_start_index)
262
263
- self.linear_method.create_weights(self,
264
- self.embedding_dim,
265
- [self.num_embeddings_per_partition],
266
- self.embedding_dim,
267
- self.num_embeddings_padded,
268
- params_dtype=params_dtype,
269
- weight_loader=self.weight_loader)
263
+ self.quant_method.create_weights(self,
264
+ self.embedding_dim,
265
+ [self.num_embeddings_per_partition],
266
+ self.embedding_dim,
267
+ self.num_embeddings_padded,
268
+ params_dtype=params_dtype,
269
+ weight_loader=self.weight_loader)
270
271
@classmethod
272
def _get_indices(cls, vocab_size_padded: int, org_vocab_size_padded: int,
@@ -412,8 +412,8 @@ class VocabParallelEmbedding(torch.nn.Module):
412
else:
413
masked_input = input_
414
# Get the embeddings.
415
- output_parallel = self.linear_method.embedding(self,
416
- masked_input.long())
415
+ output_parallel = self.quant_method.embedding(self,
416
+ masked_input.long())
417
# Mask the output embedding.
418
if self.tp_size > 1:
419
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)