@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 36a08630e8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[CORE] [QUANT] Support for GPTQModel's `dynamic` quantization per module override/control (#7086)

Qubitium-ModelCloud committed Feb 13, 2025 at 01:19 UTC 36a08630e80b6176489eef45e15b07724d95a944
8 files changed +281 -56
tests/quantization/test_gptq_dynamic.py new
+68
@@ -0,0 +1,68 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""Tests whether gptq models with dynamic quantized can be loaded.
3 +
4 +Run `pytest tests/quantization/test_gptq_dynamic.py --forked`.
5 +"""
6 +
7 +import pytest
8 +import torch
9 +
10 +from vllm.model_executor.layers.linear import UnquantizedLinearMethod
11 +from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
12 +from vllm.model_executor.layers.quantization.gptq_marlin import (
13 + GPTQMarlinLinearMethod)
14 +from vllm.model_executor.layers.quantization.utils.gptq_utils import (
15 + get_dynamic_override)
16 +
17 +PROMPT = "On the surface of Mars, we found"
18 +
19 +# The first layer is quantized using bits=4, group_size=128
20 +# The second layer is quantized using bits=8, group_size=32
21 +# All other layers (layer index >= 2) are not quantized
22 +MODEL_QUANT = [
23 + ("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
24 + True),
25 + ("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse",
26 + False),
27 +]
28 +
29 +
30 +@pytest.mark.parametrize("model_id, use_marlin_kernel", MODEL_QUANT)
31 +def test_gptq_with_dynamic(vllm_runner, model_id: str,
32 + use_marlin_kernel: bool):
33 +
34 + vllm_model = vllm_runner(model_id, dtype=torch.float16, max_model_len=2048)
35 +
36 + linear_method_cls = GPTQMarlinLinearMethod if use_marlin_kernel else (
37 + GPTQLinearMethod)
38 +
39 + for name, submodule in (vllm_model.model.llm_engine.model_executor.
40 + driver_worker.model_runner.model.named_modules()):
41 + if name == "lm_head":
42 + assert isinstance(submodule.quant_method, linear_method_cls)
43 + elif name == 'model.layers.0.self_attn.qkv_proj':
44 + # The first layer is quantized using bits=4, group_size=128
45 + # desc_act=True
46 + assert isinstance(submodule.quant_method, linear_method_cls)
47 + config = submodule.quant_method.quant_config
48 + assert config.weight_bits == 4
49 + assert config.group_size == 128
50 + assert config.desc_act
51 + elif name == 'model.layers.1.self_attn.qkv_proj':
52 + # The second layer is quantized using bits=8, group_size=32
53 + # desc_act=False
54 + assert isinstance(submodule.quant_method, linear_method_cls)
55 + config = submodule.quant_method.quant_config
56 + assert get_dynamic_override(config, layer_name=name,
57 + key="bits") == 8
58 + assert get_dynamic_override(config,
59 + layer_name=name,
60 + key="group_size") == 32
61 + assert not get_dynamic_override(
62 + config, layer_name=name, key="desc_act")
63 + elif (name == 'model.layers.2.self_attn.qkv_proj'
64 + or name == 'model.layers.2.mlp.gate_up_proj'):
65 + # All other layers (layer index >= 2) are not quantized
66 + assert isinstance(submodule.quant_method, UnquantizedLinearMethod)
67 +
68 + del vllm_model
tests/quantization/test_lm_head.py
+12 -13
@@ -3,7 +3,6 @@
3
4 Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`.
5 """
6 -from typing import Tuple
6
7 import pytest
8 import torch
@@ -17,31 +16,31 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
16
17 PROMPT = "On the surface of Mars, we found"
18
20 -MODELS_QUANT = [(
21 - "LnL-AI/TinyLlama-1.1B-intermediate-step-1341k-3T-autoround-lm_head-symFalse",
22 - True), ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
23 - ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", False)]
19 +MODELS_QUANT = [
20 + ("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head", True),
21 + ("ModelCloud/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit-10-25-2024", False),
22 + ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
23 + ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", False)
24 +]
25
26
26 -@pytest.mark.parametrize("model_lm_head_quant", MODELS_QUANT)
27 +@pytest.mark.parametrize("model_id, lm_head_quantized", MODELS_QUANT)
28 def test_lm_head(
29 vllm_runner,
29 - model_lm_head_quant: Tuple[str, bool],
30 + model_id: str,
31 + lm_head_quantized: bool,
32 ) -> None:
31 - model, lm_head_quantized = model_lm_head_quant
32 -
33 - with vllm_runner(model, dtype=torch.float16,
33 + with vllm_runner(model_id, dtype=torch.float16,
34 max_model_len=2048) as vllm_model:
35
36 def check_model(model):
37 lm_head_layer = model.lm_head
38 -
38 if lm_head_quantized:
40 - assert isinstance(lm_head_layer.linear_method,
39 + assert isinstance(lm_head_layer.quant_method,
40 (GPTQLinearMethod, GPTQMarlinLinearMethod,
41 MarlinLinearMethod))
42 else:
44 - assert isinstance(lm_head_layer.linear_method,
43 + assert isinstance(lm_head_layer.quant_method,
44 UnquantizedEmbeddingMethod)
45
46 vllm_model.apply_model(check_model)
vllm/lora/layers.py
+1 -1
@@ -1039,7 +1039,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1039 embedding_bias: Optional[torch.Tensor] = None,
1040 ) -> Optional[torch.Tensor]:
1041 # Get the logits for the next tokens.
1042 - logits = lm_head.linear_method.apply(lm_head, hidden_states)
1042 + logits = lm_head.quant_method.apply(lm_head, hidden_states)
1043 if embedding_bias is not None:
1044 logits += embedding_bias
1045
vllm/model_executor/layers/logits_processor.py
+3 -3
@@ -108,9 +108,9 @@ class LogitsProcessor(nn.Module):
108 embedding_bias: Optional[torch.Tensor],
109 ) -> Optional[torch.Tensor]:
110 # Get the logits for the next tokens.
111 - logits = lm_head.linear_method.apply(lm_head,
112 - hidden_states,
113 - bias=embedding_bias)
111 + logits = lm_head.quant_method.apply(lm_head,
112 + hidden_states,
113 + bias=embedding_bias)
114
115 # Gather logits for TP
116 logits = self._gather_logits(logits)
vllm/model_executor/layers/quantization/gptq.py
+38 -9
@@ -3,16 +3,17 @@
3 import enum
4 from enum import Enum
5 from fractions import Fraction
6 -from typing import Any, Dict, List, Optional
6 +from typing import Any, Dict, List, Optional, Union
7
8 import torch
9 from torch.nn.parameter import Parameter
10
11 from vllm import _custom_ops as ops
12 -from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12 +from vllm.model_executor.layers.linear import LinearMethodBase
13 from vllm.model_executor.layers.quantization.base_config import (
14 QuantizationConfig)
15 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15 +from vllm.model_executor.layers.quantization.utils.gptq_utils import (
16 + get_linear_quant_method)
17 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
18 GroupQuantScaleParameter,
19 PackedColumnParameter,
@@ -32,7 +33,33 @@ class GPTQConfig(QuantizationConfig):
33 group_size: int,
34 desc_act: bool,
35 lm_head_quantized: bool,
36 + dynamic: Dict[str, Dict[str, Union[int, bool]]],
37 ) -> None:
38 + # GPTQModel use `dynamic` config property to allow per module
39 + # quantization config so each module can be individually optimized.
40 + # Format is Dict[str, Dict] where key is a regex string that can
41 + # perform both positive ("+:" prefixed) or negative ("-:" prefixed)
42 + # matching of a module.
43 + # Default to positive match, override base quant config mode, if no
44 + # prefix is used. Value is in dict format of field key and override
45 + # value.
46 + # Negative matching will skip quantization init for this module
47 + # entirely:
48 + # non-quantized inference. More details and quantization examples can be
49 + # found at: https://github.com/ModelCloud/GPTQModel
50 + # Example:
51 + # # last 1/2 of the layers 10-21 has 8bit vs 4bit for 0-9
52 + # # last 1/4 of the layers 16-21 has 8bit and group_size 64
53 + # dynamic = {
54 + # #`.*\.` matches the layers_node prefix
55 + # # positive match layer 10-15
56 + # r"+:.*\.(?:1[0-5])\..*": {"bits": 8,},
57 + # # positive match layer 16-21
58 + # r"+:.*\.(?:1[6-9]|20|21)\..*": {"bits": 8, "group_size": 64,},
59 + # r"-:.*\.moe\..*": {}, # negative match (skip) all `moe` layers
60 + # }
61 + self.dynamic = dynamic
62 +
63 self.weight_bits = weight_bits
64 self.group_size = group_size
65 self.desc_act = desc_act
@@ -47,7 +74,8 @@ class GPTQConfig(QuantizationConfig):
74 return (f"GPTQConfig(weight_bits={self.weight_bits}, "
75 f"group_size={self.group_size}, "
76 f"desc_act={self.desc_act}),"
50 - f"lm_head_quantized={self.lm_head_quantized}")
77 + f"lm_head_quantized={self.lm_head_quantized}), "
78 + f"dynamic={self.dynamic}")
79
80 @classmethod
81 def get_name(cls) -> str:
@@ -68,19 +96,20 @@ class GPTQConfig(QuantizationConfig):
96
97 @classmethod
98 def from_config(cls, config: Dict[str, Any]) -> "GPTQConfig":
99 + dynamic = cls.get_from_keys_or(config, ["dynamic"], default={})
100 + dynamic = {} if dynamic is None else dynamic
101 +
102 weight_bits = cls.get_from_keys(config, ["bits"])
103 group_size = cls.get_from_keys(config, ["group_size"])
104 desc_act = cls.get_from_keys(config, ["desc_act"])
105 lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
106 default=False)
76 - return cls(weight_bits, group_size, desc_act, lm_head_quantized)
107 + return cls(weight_bits, group_size, desc_act, lm_head_quantized,
108 + dynamic)
109
110 def get_quant_method(self, layer: torch.nn.Module,
111 prefix: str) -> Optional["GPTQLinearMethod"]:
80 - if (isinstance(layer, LinearBase) or
81 - (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
82 - return GPTQLinearMethod(self)
83 - return None
112 + return get_linear_quant_method(self, layer, prefix, GPTQLinearMethod)
113
114
115 class ExllamaState(Enum):
vllm/model_executor/layers/quantization/gptq_marlin.py
+47 -12
@@ -9,17 +9,21 @@ from vllm import _custom_ops as ops
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.fused_moe.layer import (
11 FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
12 -from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
12 +from vllm.model_executor.layers.linear import (LinearMethodBase,
13 + UnquantizedLinearMethod,
14 set_weight_attrs)
15 from vllm.model_executor.layers.quantization.base_config import (
16 QuantizationConfig)
17 from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
18 MPLinearLayerConfig, choose_mp_linear_kernel)
19 from vllm.model_executor.layers.quantization.utils import replace_parameter
20 +from vllm.model_executor.layers.quantization.utils.gptq_utils import (
21 + get_linear_quant_method)
22 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
23 check_marlin_supported, marlin_moe_permute_scales,
24 marlin_repeat_scales_on_all_ranks, verify_marlin_supported)
22 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
25 +from vllm.model_executor.layers.vocab_parallel_embedding import (
26 + UnquantizedEmbeddingMethod)
27 from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
28 GroupQuantScaleParameter,
29 PackedColumnParameter,
@@ -47,12 +51,41 @@ class GPTQMarlinConfig(QuantizationConfig):
51 desc_act: bool,
52 is_sym: bool,
53 lm_head_quantized: bool,
54 + dynamic: Dict[str, Dict[str, Union[int, bool]]],
55 ) -> None:
56 if desc_act and group_size == -1:
57 # In this case, act_order == True is the same as act_order == False
58 # (since we have only one group per output channel)
59 desc_act = False
60
61 + # GPTQModel use `dynamic` config property to allow per module
62 + # quantization config so each module can be individually optimized.
63 + # Format is Dict[str, Dict] where key is a regex string that can
64 + # perform both positive ("+:" prefixed) or negative ("-:" prefixed)
65 + # matching of a module.
66 + # Default to positive match, override base quant config mode, if no
67 + # prefix is used. Value is in dict format of field key and override
68 + # value.
69 + # Negative matching will skip quantization init for this module
70 + # entirely:
71 + # non-quantized inference. More details and quantization examples can be
72 + # found at: https://github.com/ModelCloud/GPTQModel
73 + # Example:
74 + # # last 1/2 of the layers 10-21 has 8bit vs 4bit for 0-9
75 + # # last 1/4 of the layers 16-21 has 8bit and group_size 64
76 + # dynamic = {
77 + # #`.*\.` matches the layers_node prefix
78 + # # positive match layer 10-15
79 + # r"+:.*\.(?:1[0-5])\..*": {"bits": 8,},
80 + # # positive match layer 16-21
81 + # r"+:.*\.(?:1[6-9]|20|21)\..*": {"bits": 8, "group_size": 64,},
82 + # r"-:.*\.moe\..*": {}, # negative match (skip) all `moe` layers
83 + # }
84 + self.dynamic = dynamic
85 +
86 + self.weight_bits = weight_bits
87 + self.is_sym = is_sym
88 +
89 self.pack_factor = 32 // weight_bits # packed into int32
90 self.group_size = group_size
91 self.desc_act = desc_act
@@ -68,7 +101,8 @@ class GPTQMarlinConfig(QuantizationConfig):
101 return (f"GPTQMarlinConfig(quant_type={self.quant_type}, "
102 f"group_size={self.group_size}, "
103 f"desc_act={self.desc_act}, "
71 - f"lm_head_quantized={self.lm_head_quantized})")
104 + f"lm_head_quantized={self.lm_head_quantized}), "
105 + f"dynamic={self.dynamic}")
106
107 @classmethod
108 def get_name(cls) -> str:
@@ -88,6 +122,9 @@ class GPTQMarlinConfig(QuantizationConfig):
122
123 @classmethod
124 def from_config(cls, config: Dict[str, Any]) -> "GPTQMarlinConfig":
125 + dynamic = cls.get_from_keys_or(config, ["dynamic"], default={})
126 + dynamic = {} if dynamic is None else dynamic
127 +
128 weight_bits = cls.get_from_keys(config, ["bits"])
129 group_size = cls.get_from_keys(config, ["group_size"])
130 desc_act = cls.get_from_keys(config, ["desc_act"])
@@ -95,7 +132,7 @@ class GPTQMarlinConfig(QuantizationConfig):
132 lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
133 default=False)
134 return cls(weight_bits, group_size, desc_act, is_sym,
98 - lm_head_quantized)
135 + lm_head_quantized, dynamic)
136
137 @classmethod
138 def override_quantization_method(cls, hf_quant_cfg,
@@ -120,17 +157,15 @@ class GPTQMarlinConfig(QuantizationConfig):
157
158 def get_quant_method(
159 self, layer: torch.nn.Module, prefix: str
123 - ) -> Optional[Union["GPTQMarlinLinearMethod", "GPTQMarlinMoEMethod"]]:
124 - if isinstance(layer, LinearBase) or (isinstance(layer, ParallelLMHead)
125 - and self.lm_head_quantized):
126 - return GPTQMarlinLinearMethod(self)
127 - elif isinstance(layer, FusedMoE):
160 + ) -> Optional[Union["GPTQMarlinLinearMethod", "GPTQMarlinMoEMethod",
161 + UnquantizedLinearMethod, UnquantizedEmbeddingMethod]]:
162 + if isinstance(layer, FusedMoE):
163 return GPTQMarlinMoEMethod(self)
129 - return None
164 + return get_linear_quant_method(self, layer, prefix,
165 + GPTQMarlinLinearMethod)
166
167 @classmethod
168 def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]):
133 - # Extract data from quant config.
169 quant_method = quant_config.get("quant_method", "").lower()
170 num_bits = quant_config.get("bits")
171 group_size = quant_config.get("group_size")
@@ -143,7 +178,7 @@ class GPTQMarlinConfig(QuantizationConfig):
178 if quant_method != "gptq":
179 return False
180
146 - # If we cannot find the info needed in the config, cannot convert.
181 + # Marlin conversion is only valid if required properties are found
182 if (num_bits is None or group_size is None or sym is None
183 or desc_act is None):
184 return False
vllm/model_executor/layers/quantization/utils/gptq_utils.py new
+94
@@ -0,0 +1,94 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import re
3 +from copy import deepcopy
4 +from typing import Dict, Optional, Union
5 +
6 +import torch
7 +
8 +from vllm.config import QuantizationConfig
9 +from vllm.model_executor.layers.linear import (LinearBase,
10 + UnquantizedLinearMethod)
11 +from vllm.model_executor.layers.vocab_parallel_embedding import (
12 + ParallelLMHead, UnquantizedEmbeddingMethod)
13 +
14 +
15 +# Match dynamic rules with module name (prefix) and override quantize
16 +# config if module (prefix) matches a rule
17 +def override_config(config: QuantizationConfig, prefix: str):
18 + weight_bits = get_dynamic_override(config, prefix, "bits",
19 + config.weight_bits)
20 + if isinstance(weight_bits, int):
21 + config.weight_bits = weight_bits
22 + group_size = get_dynamic_override(config, prefix, "group_size",
23 + config.group_size)
24 + if isinstance(group_size, int):
25 + config.group_size = group_size
26 + desc_act = get_dynamic_override(config, prefix, "desc_act",
27 + config.desc_act)
28 + if isinstance(desc_act, bool):
29 + config.desc_act = desc_act
30 +
31 + config.pack_factor = 32 // config.weight_bits # packed into int32
32 + if config.get_name() == "gptq_marlin":
33 + is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
34 + if isinstance(is_sym, bool):
35 + config.is_sym = is_sym
36 +
37 + if (config.weight_bits, config.is_sym) not in config.TYPE_MAP:
38 + raise ValueError("Unsupported quantization config: "
39 + f"bits={config.weight_bits}, sym={config.is_sym}")
40 +
41 + config.quant_type = config.TYPE_MAP[(config.weight_bits,
42 + config.is_sym)]
43 + elif config.get_name() == "gptq":
44 + if config.weight_bits not in [2, 3, 4, 8]:
45 + raise ValueError(
46 + "Currently, only 2/3/4/8-bit weight quantization is "
47 + f"supported for GPTQ, but got {config.weight_bits} bits.")
48 +
49 +
50 +def get_dynamic_override(
51 + config: QuantizationConfig,
52 + layer_name: str,
53 + key: Optional[str] = None,
54 + default_value: Union[int, bool,
55 + None] = None) -> Union[Dict, int, bool, None]:
56 + for pattern, pattern_dict in config.dynamic.items():
57 + # Negative match: matched modules are excluded from quantized init
58 + if pattern.startswith("-:"):
59 + if re.match(pattern.removeprefix("-:"), layer_name):
60 + return False
61 + # Positive match: matched modules have quant properties overrides
62 + # base quant config
63 + elif re.match(pattern.removeprefix("+:"), layer_name):
64 + if key is None:
65 + return pattern_dict
66 + else:
67 + return pattern_dict.get(key, default_value)
68 + return default_value
69 +
70 +
71 +def get_linear_quant_method(
72 + config: QuantizationConfig,
73 + layer: torch.nn.Module,
74 + prefix: str,
75 + linear_method_cls: type,
76 +):
77 + cloned_config = deepcopy(config)
78 + parallel_lm_head_quantized = isinstance(
79 + layer, ParallelLMHead) and cloned_config.lm_head_quantized
80 + if isinstance(layer, LinearBase) or parallel_lm_head_quantized:
81 + # False = skip module, None = no override, else = Positive match
82 + if get_dynamic_override( # noqa: E712
83 + cloned_config, # noqa: E712
84 + layer_name=prefix) == False: # noqa: E712
85 + if parallel_lm_head_quantized:
86 + return UnquantizedEmbeddingMethod()
87 + return UnquantizedLinearMethod()
88 +
89 + if prefix:
90 + # Dynamic per module/layer rules may override base config
91 + override_config(cloned_config, prefix=prefix)
92 +
93 + return linear_method_cls(cloned_config)
94 + return None
vllm/model_executor/layers/vocab_parallel_embedding.py
+18 -18
@@ -226,24 +226,24 @@ class VocabParallelEmbedding(torch.nn.Module):
226 self.tp_size)
227 self.embedding_dim = embedding_dim
228
229 - linear_method = None
229 + quant_method = None
230 if quant_config is not None:
231 - linear_method = quant_config.get_quant_method(self, prefix=prefix)
232 - if linear_method is None:
233 - linear_method = UnquantizedEmbeddingMethod()
231 + quant_method = quant_config.get_quant_method(self, prefix=prefix)
232 + if quant_method is None:
233 + quant_method = UnquantizedEmbeddingMethod()
234
235 # If we are making an embedding layer, then our quantization linear
236 # method must implement the embedding operation. If we are another
237 # layer type like ParallelLMHead, this is not important.
238 is_embedding_layer = type(self.__class__) is VocabParallelEmbedding
239 - linear_method_implements_embedding = method_has_implemented_embedding(
240 - type(linear_method))
241 - if is_embedding_layer and not linear_method_implements_embedding:
239 + quant_method_implements_embedding = method_has_implemented_embedding(
240 + type(quant_method))
241 + if is_embedding_layer and not quant_method_implements_embedding:
242 raise NotImplementedError(
243 - f"The class {type(linear_method).__name__} must implement "
243 + f"The class {type(quant_method).__name__} must implement "
244 "the 'embedding' method, see UnquantizedEmbeddingMethod.")
245
246 - self.linear_method: QuantizeMethodBase = linear_method
246 + self.quant_method: QuantizeMethodBase = quant_method
247
248 if params_dtype is None:
249 params_dtype = torch.get_default_dtype()
@@ -260,13 +260,13 @@ class VocabParallelEmbedding(torch.nn.Module):
260 self.shard_indices.added_vocab_end_index -
261 self.shard_indices.added_vocab_start_index)
262
263 - self.linear_method.create_weights(self,
264 - self.embedding_dim,
265 - [self.num_embeddings_per_partition],
266 - self.embedding_dim,
267 - self.num_embeddings_padded,
268 - params_dtype=params_dtype,
269 - weight_loader=self.weight_loader)
263 + self.quant_method.create_weights(self,
264 + self.embedding_dim,
265 + [self.num_embeddings_per_partition],
266 + self.embedding_dim,
267 + self.num_embeddings_padded,
268 + params_dtype=params_dtype,
269 + weight_loader=self.weight_loader)
270
271 @classmethod
272 def _get_indices(cls, vocab_size_padded: int, org_vocab_size_padded: int,
@@ -412,8 +412,8 @@ class VocabParallelEmbedding(torch.nn.Module):
412 else:
413 masked_input = input_
414 # Get the embeddings.
415 - output_parallel = self.linear_method.embedding(self,
416 - masked_input.long())
415 + output_parallel = self.quant_method.embedding(self,
416 + masked_input.long())
417 # Mask the output embedding.
418 if self.tp_size > 1:
419 output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)