@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a62aaf1df5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc][Refactor] Generalize linear_method to be quant_method (#4373)

Cody Yu committed Apr 26, 2024 at 13:41 UTC a62aaf1df558d69658a42c1ab749368ab0325f35
45 files changed +759 -713
tests/quantization/test_fp8.py
+1 -1
@@ -20,5 +20,5 @@ def test_load_fp16_model(vllm_runner) -> None:
20
21 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
22 fc1 = model.model.decoder.layers[0].fc1
23 - assert isinstance(fc1.linear_method, Fp8LinearMethod)
23 + assert isinstance(fc1.quant_method, Fp8LinearMethod)
24 assert fc1.weight.dtype == torch.float8_e4m3fn
tests/tensorizer_loader/test_tensorizer.py
+2 -2
@@ -50,10 +50,10 @@ def test_load_with_tensorizer(mock_agent, tensorizer_config):
50 mock_agent_instance.deserialize.return_value = MagicMock()
51
52 result = load_with_tensorizer(tensorizer_config,
53 - linear_method=mock_linear_method)
53 + quant_method=mock_linear_method)
54
55 mock_agent.assert_called_once_with(tensorizer_config,
56 - linear_method=mock_linear_method)
56 + quant_method=mock_linear_method)
57 mock_agent_instance.deserialize.assert_called_once()
58 assert result == mock_agent_instance.deserialize.return_value
59
vllm/lora/layers.py
+13 -17
@@ -389,10 +389,9 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
389 self.indices = base_indices
390 self.indices_len = indices_len
391
392 - def apply_weights(self, x: torch.Tensor,
393 - bias: Optional[torch.Tensor]) -> torch.Tensor:
394 - output = self.base_layer.linear_method.apply_weights(
395 - self.base_layer, x, bias)
392 + def apply(self, x: torch.Tensor,
393 + bias: Optional[torch.Tensor]) -> torch.Tensor:
394 + output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
395 _apply_lora(
396 x,
397 self.lora_a_stacked,
@@ -416,7 +415,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
415 if not self.base_layer.skip_bias_add else None)
416
417 # Matrix multiply.
419 - output_parallel = self.apply_weights(input_, bias)
418 + output_parallel = self.apply(input_, bias)
419 if self.base_layer.gather_output:
420 # All-gather across the partitions.
421 output = tensor_model_parallel_all_gather(output_parallel)
@@ -523,10 +522,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
522 index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
523 lora_b[1].T, non_blocking=True)
524
526 - def apply_weights(self, x: torch.Tensor,
527 - bias: Optional[torch.Tensor]) -> torch.Tensor:
528 - output = self.base_layer.linear_method.apply_weights(
529 - self.base_layer, x, bias)
525 + def apply(self, x: torch.Tensor,
526 + bias: Optional[torch.Tensor]) -> torch.Tensor:
527 + output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
528 _apply_lora_packed_nslice(
529 x,
530 self.lora_a_stacked,
@@ -765,10 +763,9 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
763 index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
764 lora_a[2].T, non_blocking=True)
765
768 - def apply_weights(self, x: torch.Tensor,
769 - bias: Optional[torch.Tensor]) -> torch.Tensor:
770 - output = self.base_layer.linear_method.apply_weights(
771 - self.base_layer, x, bias)
766 + def apply(self, x: torch.Tensor,
767 + bias: Optional[torch.Tensor]) -> torch.Tensor:
768 + output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
769 _apply_lora_packed_nslice(
770 x,
771 self.lora_a_stacked,
@@ -862,9 +859,8 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
859 self.indices = base_indices
860 self.indices_len = indices_len
861
865 - def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
866 - output = self.base_layer.linear_method.apply_weights(
867 - self.base_layer, x)
862 + def apply(self, x: torch.Tensor) -> torch.Tensor:
863 + output = self.base_layer.quant_method.apply(self.base_layer, x)
864 _apply_lora(
865 x,
866 self.lora_a_stacked,
@@ -897,7 +893,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
893 input_parallel = splitted_input[tp_rank].contiguous()
894
895 # Matrix multiply.
900 - output_parallel = self.apply_weights(input_parallel)
896 + output_parallel = self.apply(input_parallel)
897 if self.base_layer.reduce_results and self.base_layer.tp_size > 1:
898 output_ = tensor_model_parallel_all_reduce(output_parallel)
899 else:
vllm/model_executor/layers/linear.py
+88 -81
@@ -1,9 +1,8 @@
1 -from abc import ABC, abstractmethod
1 +from abc import abstractmethod
2 from typing import List, Optional
3
4 import torch
5 import torch.nn.functional as F
6 -from torch import nn
6 from torch.nn.parameter import Parameter
7
8 from vllm.distributed import (divide, get_tensor_model_parallel_rank,
@@ -12,6 +11,8 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
11 tensor_model_parallel_all_gather,
12 tensor_model_parallel_all_reduce)
13 from vllm.logger import init_logger
14 +from vllm.model_executor.layers.quantization.base_config import (
15 + QuantizationConfig, QuantizeMethodBase)
16 from vllm.model_executor.utils import set_weight_attrs
17
18 logger = init_logger(__name__)
@@ -25,7 +26,7 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
26 return shard_size * marlin_tile_size, shard_offset * marlin_tile_size
27
28
28 -class LinearMethodBase(ABC):
29 +class LinearMethodBase(QuantizeMethodBase):
30 """Base class for different (maybe quantized) linear methods."""
31
32 @abstractmethod
@@ -50,22 +51,15 @@ class LinearMethodBase(ABC):
51 raise NotImplementedError
52
53 @abstractmethod
53 - def apply_weights(self,
54 - layer: torch.nn.Module,
55 - x: torch.Tensor,
56 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
54 + def apply(self,
55 + layer: torch.nn.Module,
56 + x: torch.Tensor,
57 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
58 """Apply the weights in layer to the input tensor.
59
60 Expects create_weights to have been called before on the layer."""
61 raise NotImplementedError
62
62 - def process_weights_after_loading(self, layer: nn.Module) -> None:
63 - """Process the weight after loading.
64 -
65 - This can be used for example, to transpose weights for computation.
66 - """
67 - return
68 -
63
64 class UnquantizedLinearMethod(LinearMethodBase):
65 """Linear method without quantization.
@@ -92,10 +86,10 @@ class UnquantizedLinearMethod(LinearMethodBase):
86 layer.register_parameter("weight", weight)
87 set_weight_attrs(weight, extra_weight_attrs)
88
95 - def apply_weights(self,
96 - layer: torch.nn.Module,
97 - x: torch.Tensor,
98 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
89 + def apply(self,
90 + layer: torch.nn.Module,
91 + x: torch.Tensor,
92 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
93 weight = layer.weight
94 if self.separate_bias_add:
95 if bias is not None:
@@ -104,8 +98,8 @@ class UnquantizedLinearMethod(LinearMethodBase):
98 return F.linear(x, weight, bias)
99
100
107 -class ReplicatedLinear(torch.nn.Module):
108 - """Replicated linear layer.
101 +class LinearBase(torch.nn.Module):
102 + """Base linear layer.
103
104 Args:
105 input_size: input dimension of the linear layer.
@@ -113,17 +107,16 @@ class ReplicatedLinear(torch.nn.Module):
107 bias: If true, add bias.
108 skip_bias_add: If true, skip adding bias but instead return it.
109 params_dtype: Data type for the parameters.
116 - linear_method: (Maybe quantized) linear method.
110 + quant_config: Quantization configure.
111 """
112
113 def __init__(
114 self,
115 input_size: int,
116 output_size: int,
123 - bias: bool = True,
117 skip_bias_add: bool = False,
118 params_dtype: Optional[torch.dtype] = None,
126 - linear_method: Optional[LinearMethodBase] = None,
119 + quant_config: Optional[QuantizationConfig] = None,
120 ):
121 super().__init__()
122
@@ -134,12 +127,43 @@ class ReplicatedLinear(torch.nn.Module):
127 if params_dtype is None:
128 params_dtype = torch.get_default_dtype()
129 self.params_dtype = params_dtype
137 - if linear_method is None:
138 - linear_method = UnquantizedLinearMethod()
139 - self.linear_method = linear_method
140 - self.linear_method.create_weights(self, self.input_size,
141 - [self.output_size], self.input_size,
142 - self.output_size, self.params_dtype)
130 + if quant_config is None:
131 + self.quant_method = UnquantizedLinearMethod()
132 + else:
133 + self.quant_method = quant_config.get_quant_method(self)
134 +
135 + def forward(self, x: torch.Tensor) -> torch.Tensor:
136 + raise NotImplementedError
137 +
138 +
139 +class ReplicatedLinear(LinearBase):
140 + """Replicated linear layer.
141 +
142 + Args:
143 + input_size: input dimension of the linear layer.
144 + output_size: output dimension of the linear layer.
145 + bias: If true, add bias.
146 + skip_bias_add: If true, skip adding bias but instead return it.
147 + params_dtype: Data type for the parameters.
148 + quant_config: Quantization configure.
149 + """
150 +
151 + def __init__(
152 + self,
153 + input_size: int,
154 + output_size: int,
155 + bias: bool = True,
156 + skip_bias_add: bool = False,
157 + params_dtype: Optional[torch.dtype] = None,
158 + quant_config: Optional[QuantizationConfig] = None,
159 + ):
160 + super().__init__(input_size, output_size, skip_bias_add, params_dtype,
161 + quant_config)
162 +
163 + self.quant_method.create_weights(self, self.input_size,
164 + [self.output_size], self.input_size,
165 + self.output_size, self.params_dtype)
166 +
167 if bias:
168 self.bias = Parameter(
169 torch.empty(self.output_size, dtype=self.params_dtype))
@@ -149,12 +173,12 @@ class ReplicatedLinear(torch.nn.Module):
173
174 def forward(self, x: torch.Tensor) -> torch.Tensor:
175 bias = self.bias if not self.skip_bias_add else None
152 - output = self.linear_method.apply_weights(self, x, bias)
176 + output = self.quant_method.apply(self, x, bias)
177 output_bias = self.bias if self.skip_bias_add else None
178 return output, output_bias
179
180
157 -class ColumnParallelLinear(torch.nn.Module):
181 +class ColumnParallelLinear(LinearBase):
182 """Linear layer with column parallelism.
183
184 The linear layer is defined as Y = XA + b. A is parallelized along
@@ -171,7 +195,7 @@ class ColumnParallelLinear(torch.nn.Module):
195 bias can be fused with other element-wise operations. we
196 skip adding bias but instead return it.
197 params_dtype: Data type for the parameters.
174 - linear_method: (Maybe quantized) linear method.
198 + quant_config: Quantization configure.
199 output_sizes: list of output sizes packed into one output, like for QKV
200 the list would be size 3.
201 """
@@ -184,34 +208,26 @@ class ColumnParallelLinear(torch.nn.Module):
208 gather_output: bool = False,
209 skip_bias_add: bool = False,
210 params_dtype: Optional[torch.dtype] = None,
187 - linear_method: Optional[LinearMethodBase] = None,
211 + quant_config: Optional[QuantizationConfig] = None,
212 output_sizes: Optional[List[int]] = None,
213 ):
190 - super().__init__()
214 + super().__init__(input_size, output_size, skip_bias_add, params_dtype,
215 + quant_config)
216
192 - # Keep input parameters
193 - self.input_size = input_size
194 - self.output_size = output_size
217 self.gather_output = gather_output
218 +
219 # Divide the weight matrix along the last dimension.
220 tp_size = get_tensor_model_parallel_world_size()
221 self.output_size_per_partition = divide(output_size, tp_size)
199 - self.skip_bias_add = skip_bias_add
200 - if params_dtype is None:
201 - params_dtype = torch.get_default_dtype()
202 - self.params_dtype = params_dtype
203 - if linear_method is None:
204 - linear_method = UnquantizedLinearMethod()
222 if output_sizes is None:
223 output_sizes = [output_size]
207 - self.linear_method = linear_method
208 - self.linear_method.create_weights(self,
209 - self.input_size,
210 - [x // tp_size for x in output_sizes],
211 - self.input_size,
212 - self.output_size,
213 - self.params_dtype,
214 - weight_loader=self.weight_loader)
224 + self.quant_method.create_weights(self,
225 + self.input_size,
226 + [x // tp_size for x in output_sizes],
227 + self.input_size,
228 + self.output_size,
229 + self.params_dtype,
230 + weight_loader=self.weight_loader)
231 if bias:
232 self.bias = Parameter(
233 torch.empty(self.output_size_per_partition,
@@ -239,7 +255,7 @@ class ColumnParallelLinear(torch.nn.Module):
255 bias = self.bias if not self.skip_bias_add else None
256
257 # Matrix multiply.
242 - output_parallel = self.linear_method.apply_weights(self, input_, bias)
258 + output_parallel = self.quant_method.apply(self, input_, bias)
259 if self.gather_output:
260 # All-gather across the partitions.
261 output = tensor_model_parallel_all_gather(output_parallel)
@@ -267,7 +283,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
283 bias can be fused with other element-wise operations. we
284 skip adding bias but instead return it.
285 params_dtype: Data type for the parameters.
270 - linear_method: (Maybe quantized) linear method.
286 + quant_config: Quantization configure.
287 """
288
289 def __init__(
@@ -278,13 +294,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
294 gather_output: bool = False,
295 skip_bias_add: bool = False,
296 params_dtype: Optional[torch.dtype] = None,
281 - linear_method: Optional[LinearMethodBase] = None,
297 + quant_config: Optional[QuantizationConfig] = None,
298 ):
299 self.output_sizes = output_sizes
300 tp_size = get_tensor_model_parallel_world_size()
301 assert all(output_size % tp_size == 0 for output_size in output_sizes)
302 super().__init__(input_size, sum(output_sizes), bias, gather_output,
287 - skip_bias_add, params_dtype, linear_method,
303 + skip_bias_add, params_dtype, quant_config,
304 self.output_sizes)
305
306 def weight_loader(self,
@@ -384,7 +400,7 @@ class QKVParallelLinear(ColumnParallelLinear):
400 bias can be fused with other element-wise operations. we
401 skip adding bias but instead return it.
402 params_dtype: Data type for the parameters.
387 - linear_method: (Maybe quantized) linear method.
403 + quant_config: Quantization configure.
404 """
405
406 def __init__(
@@ -396,7 +412,7 @@ class QKVParallelLinear(ColumnParallelLinear):
412 bias: bool = True,
413 skip_bias_add: bool = False,
414 params_dtype: Optional[torch.dtype] = None,
399 - linear_method: Optional[LinearMethodBase] = None,
415 + quant_config: Optional[QuantizationConfig] = None,
416 ):
417 self.hidden_size = hidden_size
418 self.head_size = head_size
@@ -424,7 +440,7 @@ class QKVParallelLinear(ColumnParallelLinear):
440 ]
441
442 super().__init__(input_size, output_size, bias, False, skip_bias_add,
427 - params_dtype, linear_method, output_sizes)
443 + params_dtype, quant_config, output_sizes)
444
445 def weight_loader(self,
446 param: Parameter,
@@ -517,7 +533,7 @@ class QKVParallelLinear(ColumnParallelLinear):
533 param_data.copy_(loaded_weight)
534
535
520 -class RowParallelLinear(torch.nn.Module):
536 +class RowParallelLinear(LinearBase):
537 """Linear layer with row parallelism.
538
539 The linear layer is defined as Y = XA + b. A is parallelized along
@@ -540,7 +556,7 @@ class RowParallelLinear(torch.nn.Module):
556 bias can be fused with other element-wise operations.
557 We skip adding bias but instead return it.
558 params_dtype: Data type for the parameters.
543 - linear_method: (Maybe quantized) linear method.
559 + quant_config: Quantization configure.
560 """
561
562 def __init__(
@@ -552,32 +568,24 @@ class RowParallelLinear(torch.nn.Module):
568 skip_bias_add: bool = False,
569 params_dtype: Optional[torch.dtype] = None,
570 reduce_results: bool = True,
555 - linear_method: Optional[LinearMethodBase] = None,
571 + quant_config: Optional[QuantizationConfig] = None,
572 ):
557 - super().__init__()
558 - # Keep input parameters
559 - self.input_size = input_size
560 - self.output_size = output_size
573 + super().__init__(input_size, output_size, skip_bias_add, params_dtype,
574 + quant_config)
575 +
576 self.input_is_parallel = input_is_parallel
577 self.reduce_results = reduce_results
563 - if params_dtype is None:
564 - params_dtype = torch.get_default_dtype()
565 - self.params_dtype = params_dtype
578
579 # Divide the weight matrix along the last dimension.
580 self.tp_size = get_tensor_model_parallel_world_size()
581 self.input_size_per_partition = divide(input_size, self.tp_size)
570 - self.skip_bias_add = skip_bias_add
571 - if linear_method is None:
572 - linear_method = UnquantizedLinearMethod()
573 - self.linear_method = linear_method
574 - self.linear_method.create_weights(self,
575 - self.input_size_per_partition,
576 - [self.output_size],
577 - self.input_size,
578 - self.output_size,
579 - self.params_dtype,
580 - weight_loader=self.weight_loader)
582 + self.quant_method.create_weights(self,
583 + self.input_size_per_partition,
584 + [self.output_size],
585 + self.input_size,
586 + self.output_size,
587 + self.params_dtype,
588 + weight_loader=self.weight_loader)
589
590 if not reduce_results and (bias and not skip_bias_add):
591 raise ValueError("When not reduce the results, adding bias to the "
@@ -616,8 +624,7 @@ class RowParallelLinear(torch.nn.Module):
624 input_parallel = splitted_input[tp_rank].contiguous()
625
626 # Matrix multiply.
619 - output_parallel = self.linear_method.apply_weights(
620 - self, input_parallel)
627 + output_parallel = self.quant_method.apply(self, input_parallel)
628 if self.reduce_results and self.tp_size > 1:
629 output_ = tensor_model_parallel_all_reduce(output_parallel)
630 else:
vllm/model_executor/layers/quantization/__init__.py
+2 -2
@@ -4,7 +4,7 @@ from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4 from vllm.model_executor.layers.quantization.awq import AWQConfig
5 from vllm.model_executor.layers.quantization.base_config import (
6 QuantizationConfig)
7 -from vllm.model_executor.layers.quantization.fp8 import FP8Config
7 +from vllm.model_executor.layers.quantization.fp8 import Fp8Config
8 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
9 from vllm.model_executor.layers.quantization.marlin import MarlinConfig
10 from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
12 QUANTIZATION_METHODS = {
13 "aqlm": AQLMConfig,
14 "awq": AWQConfig,
15 - "fp8": FP8Config,
15 + "fp8": Fp8Config,
16 "gptq": GPTQConfig,
17 "squeezellm": SqueezeLLMConfig,
18 "marlin": MarlinConfig,
vllm/model_executor/layers/quantization/aqlm.py
+8 -5
@@ -9,10 +9,10 @@ import torch.nn.functional as F
9 from torch.nn.parameter import Parameter
10
11 from vllm import _custom_ops as ops
12 -from vllm.model_executor.layers.linear import (LinearMethodBase,
13 - set_weight_attrs)
12 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
13 from vllm.model_executor.layers.quantization.base_config import (
14 QuantizationConfig)
15 +from vllm.model_executor.utils import set_weight_attrs
16
17
18 def get_int_dtype(nbits: int) -> torch.dtype:
@@ -207,8 +207,11 @@ class AQLMConfig(QuantizationConfig):
207 return cls(in_group_size, nbits_per_codebook, num_code_books,
208 out_group_size)
209
210 - def get_linear_method(self) -> "AQLMLinearMethod":
211 - return AQLMLinearMethod(self)
210 + def get_quant_method(
211 + self, layer: torch.nn.Module) -> Optional["AQLMLinearMethod"]:
212 + if isinstance(layer, LinearBase):
213 + return AQLMLinearMethod(self)
214 + return None
215
216 def get_scaled_act_names(self) -> List[str]:
217 return []
@@ -321,7 +324,7 @@ class AQLMLinearMethod(LinearMethodBase):
324 layer.register_parameter("scales", scales)
325 set_weight_attrs(scales, extra_weight_attrs)
326
324 - def apply_weights(
327 + def apply(
328 self,
329 layer: torch.nn.Module,
330 x: torch.Tensor,
vllm/model_executor/layers/quantization/awq.py
+11 -8
@@ -4,10 +4,10 @@ import torch
4 from torch.nn.parameter import Parameter
5
6 from vllm import _custom_ops as ops
7 -from vllm.model_executor.layers.linear import (LinearMethodBase,
8 - set_weight_attrs)
7 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 from vllm.model_executor.layers.quantization.base_config import (
9 QuantizationConfig)
10 +from vllm.model_executor.utils import set_weight_attrs
11
12
13 class AWQConfig(QuantizationConfig):
@@ -62,8 +62,11 @@ class AWQConfig(QuantizationConfig):
62 zero_point = cls.get_from_keys(config, ["zero_point"])
63 return cls(weight_bits, group_size, zero_point)
64
65 - def get_linear_method(self) -> "AWQLinearMethod":
66 - return AWQLinearMethod(self)
65 + def get_quant_method(
66 + self, layer: torch.nn.Module) -> Optional["AWQLinearMethod"]:
67 + if isinstance(layer, LinearBase):
68 + return AWQLinearMethod(self)
69 + return None
70
71 def get_scaled_act_names(self) -> List[str]:
72 return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
@@ -147,10 +150,10 @@ class AWQLinearMethod(LinearMethodBase):
150 layer.register_parameter("scales", scales)
151 set_weight_attrs(scales, extra_weight_attrs)
152
150 - def apply_weights(self,
151 - layer: torch.nn.Module,
152 - x: torch.Tensor,
153 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
153 + def apply(self,
154 + layer: torch.nn.Module,
155 + x: torch.Tensor,
156 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
157 qweight = layer.qweight
158 scales = layer.scales
159 qzeros = layer.qzeros
vllm/model_executor/layers/quantization/base_config.py
+28 -3
@@ -2,8 +2,33 @@ from abc import ABC, abstractmethod
2 from typing import Any, Dict, List
3
4 import torch
5 +from torch import nn
6
6 -from vllm.model_executor.layers.linear import LinearMethodBase
7 +
8 +class QuantizeMethodBase(ABC):
9 + """Base class for different quantized methods."""
10 +
11 + @abstractmethod
12 + def create_weights(self, layer: torch.nn.Module, *weight_args,
13 + **extra_weight_attrs):
14 + """Create weights for a layer.
15 +
16 + The weights will be set as attributes of the layer."""
17 + raise NotImplementedError
18 +
19 + @abstractmethod
20 + def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
21 + """Apply the weights in layer to the input tensor.
22 +
23 + Expects create_weights to have been called before on the layer."""
24 + raise NotImplementedError
25 +
26 + def process_weights_after_loading(self, layer: nn.Module) -> None:
27 + """Process the weight after loading.
28 +
29 + This can be used for example, to transpose weights for computation.
30 + """
31 + return
32
33
34 class QuantizationConfig(ABC):
@@ -51,8 +76,8 @@ class QuantizationConfig(ABC):
76 "quantization config.")
77
78 @abstractmethod
54 - def get_linear_method(self) -> LinearMethodBase:
55 - """Get the linear method to use for the quantized linear layer."""
79 + def get_quant_method(self, layer: torch.nn.Module) -> QuantizeMethodBase:
80 + """Get the quantize method to use for the quantized layer."""
81 raise NotImplementedError
82
83 @abstractmethod
vllm/model_executor/layers/quantization/fp8.py
+20 -40
@@ -1,16 +1,17 @@
1 -from typing import Any, Dict, List, Optional, Tuple
1 +from typing import Any, Dict, List, Optional
2
3 import torch
4 from torch.nn import Module
5 from torch.nn.parameter import Parameter
6
7 -from vllm.model_executor.layers.linear import (LinearMethodBase,
8 - set_weight_attrs)
7 +from vllm import _custom_ops as ops
8 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9 from vllm.model_executor.layers.quantization.base_config import (
10 - QuantizationConfig)
10 + QuantizationConfig, QuantizeMethodBase)
11 +from vllm.model_executor.utils import set_weight_attrs
12
13
13 -class FP8Config(QuantizationConfig):
14 +class Fp8Config(QuantizationConfig):
15 """Config class for FP8."""
16
17 @classmethod
@@ -33,11 +34,14 @@ class FP8Config(QuantizationConfig):
34 return []
35
36 @classmethod
36 - def from_config(cls, config: Dict[str, Any]) -> "FP8Config":
37 + def from_config(cls, config: Dict[str, Any]) -> "Fp8Config":
38 return cls()
39
39 - def get_linear_method(self) -> "Fp8LinearMethod":
40 - return Fp8LinearMethod(self)
40 + def get_quant_method(
41 + self, layer: torch.nn.Module) -> Optional["QuantizeMethodBase"]:
42 + if isinstance(layer, LinearBase):
43 + return Fp8LinearMethod(self)
44 + return None
45
46 def get_scaled_act_names(self) -> List[str]:
47 return []
@@ -57,7 +61,7 @@ class Fp8LinearMethod(LinearMethodBase):
61 quant_config: The quantization config.
62 """
63
60 - def __init__(self, quant_config: FP8Config):
64 + def __init__(self, quant_config: Fp8Config):
65 self.quant_config = quant_config
66
67 def create_weights(
@@ -86,24 +90,24 @@ class Fp8LinearMethod(LinearMethodBase):
90 layer.register_parameter("weight_scaling_factor", w_scale)
91
92 def process_weights_after_loading(self, layer: Module) -> None:
89 - # Although the linear_method is propagated to all layers,
93 + # Although the quant_method is propagated to all layers,
94 # only linear layers invoke "create_weights". So we check
95 # whether "weight_scaling_facor" is registered to determine
96 # whether the layer is a linear layer that requires quantization.
97 if not hasattr(layer, "weight_scaling_factor"):
98 return
99
96 - qweight, weight_scale = per_tensor_quantize(layer.weight)
100 + qweight, weight_scale = ops.scaled_fp8_quant(layer.weight)
101 # torch._scaled_mm requires column-major in the second
102 # input (weight), so we transpose the quantized weight.
103 layer.weight = Parameter(qweight.t(), requires_grad=False)
104 layer.weight_scaling_factor.data.copy_(weight_scale)
105
102 - def apply_weights(self,
103 - layer: torch.nn.Module,
104 - x: torch.Tensor,
105 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
106 - qinput, x_scale = per_tensor_quantize(x)
106 + def apply(self,
107 + layer: torch.nn.Module,
108 + x: torch.Tensor,
109 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
110 + qinput, x_scale = ops.scaled_fp8_quant(x)
111 output, _ = torch._scaled_mm(
112 qinput,
113 layer.weight,
@@ -113,27 +117,3 @@ class Fp8LinearMethod(LinearMethodBase):
117 bias=bias,
118 )
119 return output
116 -
117 -
118 -def per_tensor_quantize(tensor: torch.Tensor) -> Tuple[torch.Tensor, float]:
119 - """Quantize a tensor using per-tensor static scaling factor.
120 -
121 - Args:
122 - tensor: The input tensor.
123 - """
124 - finfo = torch.finfo(torch.float8_e4m3fn)
125 - # Calculate the scale as dtype max divided by absmax.
126 - # Since .abs() creates a new tensor, we use aminmax to get
127 - # the min and max first and then calculate the absmax.
128 - min_val, max_val = tensor.aminmax()
129 - amax = min_val.abs().max(max_val.abs())
130 - scale = finfo.max / amax.clamp(min=1e-12)
131 - # scale and clamp the tensor to bring it to
132 - # the representative range of float8 data type
133 - # (as default cast is unsaturated)
134 - qweight = (tensor * scale).clamp(min=finfo.min, max=finfo.max)
135 - # Return both float8 data and the inverse scale (as float),
136 - # as both required as inputs to torch._scaled_mm
137 - qweight = qweight.to(torch.float8_e4m3fn)
138 - scale = scale.float().reciprocal()
139 - return qweight, scale
vllm/model_executor/layers/quantization/gptq.py
+11 -8
@@ -7,10 +7,10 @@ import torch
7 from torch.nn.parameter import Parameter
8
9 from vllm import _custom_ops as ops
10 -from vllm.model_executor.layers.linear import (LinearMethodBase,
11 - set_weight_attrs)
10 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 +from vllm.model_executor.utils import set_weight_attrs
14
15
16 class GPTQConfig(QuantizationConfig):
@@ -63,8 +63,11 @@ class GPTQConfig(QuantizationConfig):
63 desc_act = cls.get_from_keys(config, ["desc_act"])
64 return cls(weight_bits, group_size, desc_act)
65
66 - def get_linear_method(self) -> "GPTQLinearMethod":
67 - return GPTQLinearMethod(self)
66 + def get_quant_method(
67 + self, layer: torch.nn.Module) -> Optional["GPTQLinearMethod"]:
68 + if isinstance(layer, LinearBase):
69 + return GPTQLinearMethod(self)
70 + return None
71
72 def get_scaled_act_names(self) -> List[str]:
73 return []
@@ -194,10 +197,10 @@ class GPTQLinearMethod(LinearMethodBase):
197
198 layer.exllama_state = exllama_state
199
197 - def apply_weights(self,
198 - layer: torch.nn.Module,
199 - x: torch.Tensor,
200 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
200 + def apply(self,
201 + layer: torch.nn.Module,
202 + x: torch.Tensor,
203 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
204 qweight = layer.qweight
205 out_shape = x.shape[:-1] + (qweight.shape[-1], )
206 reshaped_x = x.reshape(-1, x.shape[-1])
vllm/model_executor/layers/quantization/marlin.py
+8 -5
@@ -4,10 +4,10 @@ import torch
4 from torch.nn.parameter import Parameter
5
6 from vllm import _custom_ops as ops
7 -from vllm.model_executor.layers.linear import (LinearMethodBase,
8 - set_weight_attrs)
7 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 from vllm.model_executor.layers.quantization.base_config import (
9 QuantizationConfig)
10 +from vllm.model_executor.utils import set_weight_attrs
11
12
13 class MarlinConfig(QuantizationConfig):
@@ -72,8 +72,11 @@ class MarlinConfig(QuantizationConfig):
72 group_size = cls.get_from_keys(config, ["group_size"])
73 return cls(group_size)
74
75 - def get_linear_method(self) -> "MarlinLinearMethod":
76 - return MarlinLinearMethod(self)
75 + def get_quant_method(
76 + self, layer: torch.nn.Module) -> Optional["MarlinLinearMethod"]:
77 + if isinstance(layer, LinearBase):
78 + return MarlinLinearMethod(self)
79 + return None
80
81 def get_scaled_act_names(self) -> List[str]:
82 return []
@@ -197,7 +200,7 @@ class MarlinLinearMethod(LinearMethodBase):
200 layer.register_parameter("workspace", workspace)
201 set_weight_attrs(workspace, extra_weight_attrs)
202
200 - def apply_weights(
203 + def apply(
204 self,
205 layer: torch.nn.Module,
206 x: torch.Tensor,
vllm/model_executor/layers/quantization/squeezellm.py
+14 -10
@@ -4,10 +4,10 @@ import torch
4 from torch.nn.parameter import Parameter
5
6 from vllm import _custom_ops as ops
7 -from vllm.model_executor.layers.linear import (LinearMethodBase,
8 - set_weight_attrs)
7 +from vllm.model_executor.layers.linear import LinearBase
8 from vllm.model_executor.layers.quantization.base_config import (
10 - QuantizationConfig)
9 + QuantizationConfig, QuantizeMethodBase)
10 +from vllm.model_executor.utils import set_weight_attrs
11 from vllm.utils import is_hip
12
13
@@ -51,14 +51,18 @@ class SqueezeLLMConfig(QuantizationConfig):
51 weight_bits = cls.get_from_keys(config, ["wbits"])
52 return cls(weight_bits)
53
54 - def get_linear_method(self) -> "SqueezeLLMLinearMethod":
55 - return SqueezeLLMLinearMethod(self)
54 + def get_quant_method(
55 + self,
56 + layer: torch.nn.Module) -> Optional["SqueezeLLMLinearMethod"]:
57 + if isinstance(layer, LinearBase):
58 + return SqueezeLLMLinearMethod(self)
59 + return
60
61 def get_scaled_act_names(self) -> List[str]:
62 return []
63
64
61 -class SqueezeLLMLinearMethod(LinearMethodBase):
65 +class SqueezeLLMLinearMethod(QuantizeMethodBase):
66 """Linear method for SqueezeLLM.
67
68 Args:
@@ -112,10 +116,10 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
116 layer.register_parameter("lookup_table", lookup_table)
117 set_weight_attrs(lookup_table, extra_weight_attrs)
118
115 - def apply_weights(self,
116 - layer: torch.nn.Module,
117 - x: torch.Tensor,
118 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
119 + def apply(self,
120 + layer: torch.nn.Module,
121 + x: torch.Tensor,
122 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
123 qweight = layer.qweight
124 lookup_table = layer.lookup_table
125 out_shape = x.shape[:-1] + (qweight.shape[-1], )
vllm/model_executor/model_loader/loader.py
+18 -20
@@ -3,8 +3,7 @@ import copy
3 import glob
4 import os
5 from abc import ABC, abstractmethod
6 -from typing import (TYPE_CHECKING, Any, Dict, Generator, List, Optional, Tuple,
7 - Type)
6 +from typing import Any, Dict, Generator, List, Optional, Tuple, Type
7
8 import torch
9 from torch import nn
@@ -13,6 +12,8 @@ from vllm.config import (VLLM_USE_MODELSCOPE, DeviceConfig, LoadConfig,
12 LoadFormat, LoRAConfig, ModelConfig, ParallelConfig,
13 SchedulerConfig, VisionLanguageConfig)
14 from vllm.logger import init_logger
15 +from vllm.model_executor.layers.quantization.base_config import (
16 + QuantizationConfig)
17 from vllm.model_executor.model_loader.tensorizer import (
18 TensorizerConfig, is_vllm_serialized_tensorizer, load_with_tensorizer,
19 tensorizer_weights_iterator)
@@ -24,9 +25,6 @@ from vllm.model_executor.model_loader.weight_utils import (
25 pt_weights_iterator, safetensors_weights_iterator)
26 from vllm.model_executor.models.llava import LlavaForConditionalGeneration
27
27 -if TYPE_CHECKING:
28 - from vllm.model_executor.layers.linear import LinearMethodBase
29 -
28 _VISION_MODEL_CLASSES = [
29 LlavaForConditionalGeneration,
30 ]
@@ -34,11 +32,10 @@ _VISION_MODEL_CLASSES = [
32 logger = init_logger(__name__)
33
34
37 -def _get_linear_method(
35 +def _get_quantization_config(
36 model_config: ModelConfig,
39 - load_config: LoadConfig) -> Optional["LinearMethodBase"]:
40 - """Get the (maybe quantized) linear method."""
41 - linear_method = None
37 + load_config: LoadConfig) -> Optional[QuantizationConfig]:
38 + """Get the quantization config."""
39 if model_config.quantization is not None:
40 quant_config = get_quant_config(model_config, load_config)
41 capability = torch.cuda.get_device_capability()
@@ -55,9 +52,8 @@ def _get_linear_method(
52 f"{model_config.dtype} is not supported for quantization "
53 f"method {model_config.quantization}. Supported dtypes: "
54 f"{supported_dtypes}")
58 -
59 - linear_method = quant_config.get_linear_method()
60 - return linear_method
55 + return quant_config
56 + return None
57
58
59 def _get_model_initialization_kwargs(
@@ -85,10 +81,10 @@ def _initialize_model(
81 vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
82 """Initialize a model with the given configurations."""
83 model_class = get_model_architecture(model_config)[0]
88 - linear_method = _get_linear_method(model_config, load_config)
84 + quant_config = _get_quantization_config(model_config, load_config)
85
86 return model_class(config=model_config.hf_config,
91 - linear_method=linear_method,
87 + quant_config=quant_config,
88 **_get_model_initialization_kwargs(
89 model_class, lora_config, vision_language_config))
90
@@ -229,9 +225,11 @@ class DefaultModelLoader(BaseModelLoader):
225 "fall_back_to_pt_during_load",
226 True)), )
227 for _, module in model.named_modules():
232 - linear_method = getattr(module, "linear_method", None)
233 - if linear_method is not None:
234 - linear_method.process_weights_after_loading(module)
228 + quant_method = getattr(module, "quant_method", None)
229 + if quant_method is not None:
230 + quant_method.process_weights_after_loading(module)
231 + # FIXME: Remove this after Mixtral is updated
232 + # to use quant_method.
233 if hasattr(module, "process_weights_after_loading"):
234 module.process_weights_after_loading()
235 return model.eval()
@@ -314,11 +312,11 @@ class TensorizerLoader(BaseModelLoader):
312 with set_default_torch_dtype(model_config.dtype):
313 with torch.device(device_config.device):
314 model_class = get_model_architecture(model_config)[0]
317 - linear_method = _get_linear_method(model_config,
318 - self.load_config)
315 + quant_config = _get_quantization_config(
316 + model_config, self.load_config)
317 extra_kwargs = _get_model_initialization_kwargs(
318 model_class, lora_config, vision_language_config)
321 - extra_kwargs["linear_method"] = linear_method
319 + extra_kwargs["quant_config"] = quant_config
320
321 tensorizer_config = copy.copy(self.tensorizer_config)
322 tensorizer_config.model_class = model_class
vllm/model_executor/model_loader/tensorizer.py
+7 -6
@@ -13,7 +13,8 @@ from transformers import PretrainedConfig
13
14 from vllm.config import ModelConfig, ParallelConfig
15 from vllm.logger import init_logger
16 -from vllm.model_executor.layers.linear import LinearMethodBase
16 +from vllm.model_executor.layers.quantization.base_config import (
17 + QuantizationConfig)
18 from vllm.model_executor.layers.vocab_parallel_embedding import (
19 VocabParallelEmbedding)
20
@@ -251,7 +252,7 @@ class TensorizerAgent:
252 """
253
254 def __init__(self, tensorizer_config: TensorizerConfig,
254 - linear_method: LinearMethodBase, **extra_kwargs):
255 + quant_config: QuantizationConfig, **extra_kwargs):
256 if tensorizer_load_fail is not None:
257 raise ImportError(
258 "Tensorizer is not installed. Please install tensorizer "
@@ -262,10 +263,10 @@ class TensorizerAgent:
263 self.tensorizer_args = (
264 self.tensorizer_config._construct_tensorizer_args())
265 self.extra_kwargs = extra_kwargs
265 - if extra_kwargs.get("linear_method", None) is not None:
266 - self.linear_method = extra_kwargs["linear_method"]
266 + if extra_kwargs.get("quant_config", None) is not None:
267 + self.quant_config = extra_kwargs["quant_config"]
268 else:
268 - self.linear_method = linear_method
269 + self.quant_config = quant_config
270 self.model = self._init_model()
271
272 def _init_model(self):
@@ -274,7 +275,7 @@ class TensorizerAgent:
275 with no_init_or_tensor():
276 return self.tensorizer_config.model_class(
277 config=model_args,
277 - linear_method=self.linear_method,
278 + quant_config=self.quant_config,
279 **self.extra_kwargs)
280
281 def _resize_lora_embeddings(self):
vllm/model_executor/models/baichuan.py
+22 -21
@@ -31,11 +31,12 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
31 get_tensor_model_parallel_world_size)
32 from vllm.model_executor.layers.activation import SiluAndMul
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 -from vllm.model_executor.layers.linear import (LinearMethodBase,
35 - MergedColumnParallelLinear,
34 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
35 QKVParallelLinear,
36 RowParallelLinear)
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 +from vllm.model_executor.layers.quantization.base_config import (
39 + QuantizationConfig)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -77,17 +78,17 @@ class BaiChuanMLP(nn.Module):
78 hidden_size: int,
79 intermediate_size: int,
80 hidden_act: str,
80 - linear_method: Optional[LinearMethodBase] = None,
81 + quant_config: Optional[QuantizationConfig] = None,
82 ):
83 super().__init__()
84 self.gate_up_proj = MergedColumnParallelLinear(
85 hidden_size, [intermediate_size] * 2,
86 bias=False,
86 - linear_method=linear_method)
87 + quant_config=quant_config)
88 self.down_proj = RowParallelLinear(intermediate_size,
89 hidden_size,
90 bias=False,
90 - linear_method=linear_method)
91 + quant_config=quant_config)
92 if hidden_act != "silu":
93 raise ValueError(f"Unsupported activation: {hidden_act}. "
94 "Only silu is supported for now.")
@@ -110,7 +111,7 @@ class BaiChuanAttention(nn.Module):
111 position_embedding: str,
112 rope_theta: float = 10000,
113 max_position_embeddings: int = 8192,
113 - linear_method: Optional[LinearMethodBase] = None,
114 + quant_config: Optional[QuantizationConfig] = None,
115 ):
116 super().__init__()
117 self.hidden_size = hidden_size
@@ -132,13 +133,13 @@ class BaiChuanAttention(nn.Module):
133 self.total_num_heads,
134 self.total_num_heads,
135 bias=False,
135 - linear_method=linear_method,
136 + quant_config=quant_config,
137 )
138 self.o_proj = RowParallelLinear(
139 self.total_num_heads * self.head_dim,
140 hidden_size,
141 bias=False,
141 - linear_method=linear_method,
142 + quant_config=quant_config,
143 )
144 # Create the alibi slopes and slice them.
145 if self.postion_embedding == "ALIBI":
@@ -184,7 +185,7 @@ class BaiChuanDecoderLayer(nn.Module):
185 def __init__(self,
186 config: PretrainedConfig,
187 position_embedding: str,
187 - linear_method: Optional[LinearMethodBase] = None):
188 + quant_config: Optional[QuantizationConfig] = None):
189 super().__init__()
190 self.hidden_size = config.hidden_size
191 rope_theta = getattr(config, "rope_theta", 10000)
@@ -196,13 +197,13 @@ class BaiChuanDecoderLayer(nn.Module):
197 position_embedding=position_embedding,
198 rope_theta=rope_theta,
199 max_position_embeddings=max_position_embeddings,
199 - linear_method=linear_method,
200 + quant_config=quant_config,
201 )
202 self.mlp = BaiChuanMLP(
203 hidden_size=self.hidden_size,
204 intermediate_size=config.intermediate_size,
205 hidden_act=config.hidden_act,
205 - linear_method=linear_method,
206 + quant_config=quant_config,
207 )
208 self.input_layernorm = RMSNorm(config.hidden_size,
209 eps=config.rms_norm_eps)
@@ -243,7 +244,7 @@ class BaiChuanModel(nn.Module):
244 def __init__(self,
245 config: PretrainedConfig,
246 position_embedding: str,
246 - linear_method: Optional[LinearMethodBase] = None):
247 + quant_config: Optional[QuantizationConfig] = None):
248 super().__init__()
249 self.config = config
250 self.padding_idx = config.pad_token_id
@@ -254,7 +255,7 @@ class BaiChuanModel(nn.Module):
255 config.hidden_size,
256 )
257 self.layers = nn.ModuleList([
257 - BaiChuanDecoderLayer(config, position_embedding, linear_method)
258 + BaiChuanDecoderLayer(config, position_embedding, quant_config)
259 for _ in range(config.num_hidden_layers)
260 ])
261 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -303,13 +304,13 @@ class BaiChuanBaseForCausalLM(nn.Module):
304 self,
305 config,
306 position_embedding: str,
306 - linear_method: Optional[LinearMethodBase] = None,
307 + quant_config: Optional[QuantizationConfig] = None,
308 lora_config: Optional[LoRAConfig] = None,
309 ):
310 super().__init__()
311 self.config = config
311 - self.linear_method = linear_method
312 - self.model = BaiChuanModel(config, position_embedding, linear_method)
312 + self.quant_config = quant_config
313 + self.model = BaiChuanModel(config, position_embedding, quant_config)
314 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
315 self.logits_processor = LogitsProcessor(config.vocab_size)
316 self.sampler = Sampler()
@@ -388,13 +389,13 @@ class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
389 def __init__(
390 self,
391 config,
391 - linear_method: Optional[LinearMethodBase] = None,
392 + quant_config: Optional[QuantizationConfig] = None,
393 lora_config: Optional[LoRAConfig] = None,
394 ):
395 if config.hidden_size == 4096: # baichuan2 7b
395 - super().__init__(config, "ROPE", linear_method, lora_config)
396 + super().__init__(config, "ROPE", quant_config, lora_config)
397 else: # baichuan 13b, baichuan2 13b
397 - super().__init__(config, "ALIBI", linear_method, lora_config)
398 + super().__init__(config, "ALIBI", quant_config, lora_config)
399
400
401 class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
@@ -403,7 +404,7 @@ class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
404 def __init__(
405 self,
406 config,
406 - linear_method: Optional[LinearMethodBase] = None,
407 + quant_config: Optional[QuantizationConfig] = None,
408 lora_config: Optional[LoRAConfig] = None,
409 ):
409 - super().__init__(config, "ROPE", linear_method, lora_config)
410 + super().__init__(config, "ROPE", quant_config, lora_config)
vllm/model_executor/models/bloom.py
+17 -16
@@ -28,10 +28,11 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
28 get_tensor_model_parallel_world_size)
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 - LinearMethodBase,
31 QKVParallelLinear,
32 RowParallelLinear)
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 +from vllm.model_executor.layers.quantization.base_config import (
35 + QuantizationConfig)
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
@@ -70,7 +71,7 @@ class BloomAttention(nn.Module):
71 def __init__(
72 self,
73 config: BloomConfig,
73 - linear_method: Optional[LinearMethodBase] = None,
74 + quant_config: Optional[QuantizationConfig] = None,
75 ):
76 super().__init__()
77 self.hidden_size = config.hidden_size
@@ -87,13 +88,13 @@ class BloomAttention(nn.Module):
88 self.head_dim,
89 self.total_num_heads,
90 bias=True,
90 - linear_method=linear_method,
91 + quant_config=quant_config,
92 )
93 self.dense = RowParallelLinear(
94 self.hidden_size,
95 self.hidden_size,
96 bias=True,
96 - linear_method=linear_method,
97 + quant_config=quant_config,
98 )
99
100 # Create the alibi slopes and slice them.
@@ -129,21 +130,21 @@ class BloomMLP(nn.Module):
130 def __init__(
131 self,
132 config: BloomConfig,
132 - linear_method: Optional[LinearMethodBase] = None,
133 + quant_config: Optional[QuantizationConfig] = None,
134 ):
135 super().__init__()
136 hidden_size = config.hidden_size
137 self.dense_h_to_4h = ColumnParallelLinear(
138 hidden_size,
139 4 * hidden_size,
139 - linear_method=linear_method,
140 + quant_config=quant_config,
141 )
141 - quant_config = getattr(linear_method, "quant_config", None)
142 + quant_config = getattr(quant_config, "quant_config", None)
143 self.gelu_impl = get_act_fn("gelu", quant_config, 4 * hidden_size)
144 self.dense_4h_to_h = RowParallelLinear(
145 4 * hidden_size,
146 hidden_size,
146 - linear_method=linear_method,
147 + quant_config=quant_config,
148 )
149
150 def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -158,17 +159,17 @@ class BloomBlock(nn.Module):
159 def __init__(
160 self,
161 config: BloomConfig,
161 - linear_method: Optional[LinearMethodBase] = None,
162 + quant_config: Optional[QuantizationConfig] = None,
163 ):
164 super().__init__()
165 hidden_size = config.hidden_size
166
167 self.input_layernorm = nn.LayerNorm(hidden_size,
168 eps=config.layer_norm_epsilon)
168 - self.self_attention = BloomAttention(config, linear_method)
169 + self.self_attention = BloomAttention(config, quant_config)
170 self.post_attention_layernorm = nn.LayerNorm(
171 hidden_size, eps=config.layer_norm_epsilon)
171 - self.mlp = BloomMLP(config, linear_method)
172 + self.mlp = BloomMLP(config, quant_config)
173 self.apply_residual_connection_post_layernorm = (
174 config.apply_residual_connection_post_layernorm)
175
@@ -214,7 +215,7 @@ class BloomModel(nn.Module):
215 def __init__(
216 self,
217 config: BloomConfig,
217 - linear_method: Optional[LinearMethodBase] = None,
218 + quant_config: Optional[QuantizationConfig] = None,
219 ):
220 super().__init__()
221 self.embed_dim = config.hidden_size
@@ -229,7 +230,7 @@ class BloomModel(nn.Module):
230
231 # Transformer blocks
232 self.h = nn.ModuleList([
232 - BloomBlock(config, linear_method)
233 + BloomBlock(config, quant_config)
234 for _ in range(config.num_hidden_layers)
235 ])
236
@@ -262,12 +263,12 @@ class BloomForCausalLM(nn.Module):
263 def __init__(
264 self,
265 config: BloomConfig,
265 - linear_method: Optional[LinearMethodBase] = None,
266 + quant_config: Optional[QuantizationConfig] = None,
267 ):
268 super().__init__()
269 self.config = config
269 - self.linear_method = linear_method
270 - self.transformer = BloomModel(config, linear_method)
270 + self.quant_config = quant_config
271 + self.transformer = BloomModel(config, quant_config)
272 self.lm_head_weight = self.transformer.word_embeddings.weight
273 self.logits_processor = LogitsProcessor(config.vocab_size)
274 self.sampler = Sampler()
vllm/model_executor/models/chatglm.py
+19 -18
@@ -13,11 +13,12 @@ from vllm.config import LoRAConfig
13 from vllm.distributed import get_tensor_model_parallel_world_size
14 from vllm.model_executor.layers.activation import SiluAndMul
15 from vllm.model_executor.layers.layernorm import RMSNorm
16 -from vllm.model_executor.layers.linear import (LinearMethodBase,
17 - MergedColumnParallelLinear,
16 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
17 QKVParallelLinear,
18 RowParallelLinear)
19 from vllm.model_executor.layers.logits_processor import LogitsProcessor
20 +from vllm.model_executor.layers.quantization.base_config import (
21 + QuantizationConfig)
22 from vllm.model_executor.layers.rotary_embedding import get_rope
23 from vllm.model_executor.layers.sampler import Sampler
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -33,7 +34,7 @@ class GLMAttention(nn.Module):
34 def __init__(
35 self,
36 config,
36 - linear_method: Optional[LinearMethodBase] = None,
37 + quant_config: Optional[QuantizationConfig] = None,
38 ):
39 super().__init__()
40 self.hidden_size = config.hidden_size
@@ -65,13 +66,13 @@ class GLMAttention(nn.Module):
66 self.total_num_heads,
67 self.total_num_kv_heads,
68 bias=config.add_bias_linear or config.add_qkv_bias,
68 - linear_method=linear_method,
69 + quant_config=quant_config,
70 )
71 self.dense = RowParallelLinear(
72 self.total_num_heads * self.head_dim,
73 config.hidden_size,
74 bias=config.add_bias_linear,
74 - linear_method=linear_method,
75 + quant_config=quant_config,
76 )
77
78 # https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
@@ -123,7 +124,7 @@ class GLMMLP(nn.Module):
124 def __init__(
125 self,
126 config,
126 - linear_method: Optional[LinearMethodBase] = None,
127 + quant_config: Optional[QuantizationConfig] = None,
128 ):
129 super().__init__()
130
@@ -134,7 +135,7 @@ class GLMMLP(nn.Module):
135 config.hidden_size,
136 [config.ffn_hidden_size] * 2,
137 bias=config.add_bias_linear,
137 - linear_method=linear_method,
138 + quant_config=quant_config,
139 )
140
141 self.activation_func = SiluAndMul()
@@ -144,7 +145,7 @@ class GLMMLP(nn.Module):
145 config.ffn_hidden_size,
146 config.hidden_size,
147 bias=config.add_bias_linear,
147 - linear_method=linear_method,
148 + quant_config=quant_config,
149 )
150
151 def forward(self, hidden_states):
@@ -166,7 +167,7 @@ class GLMBlock(nn.Module):
167 def __init__(
168 self,
169 config,
169 - linear_method: Optional[LinearMethodBase] = None,
170 + quant_config: Optional[QuantizationConfig] = None,
171 ):
172 super().__init__()
173 self.apply_residual_connection_post_layernorm = (
@@ -180,7 +181,7 @@ class GLMBlock(nn.Module):
181 eps=config.layernorm_epsilon)
182
183 # Self attention.
183 - self.self_attention = GLMAttention(config, linear_method)
184 + self.self_attention = GLMAttention(config, quant_config)
185 self.hidden_dropout = config.hidden_dropout
186
187 # Layernorm on the attention output
@@ -188,7 +189,7 @@ class GLMBlock(nn.Module):
189 config.hidden_size, eps=config.layernorm_epsilon)
190
191 # MLP
191 - self.mlp = GLMMLP(config, linear_method)
192 + self.mlp = GLMMLP(config, quant_config)
193
194 def forward(
195 self,
@@ -236,7 +237,7 @@ class GLMTransformer(nn.Module):
237 def __init__(
238 self,
239 config,
239 - linear_method: Optional[LinearMethodBase] = None,
240 + quant_config: Optional[QuantizationConfig] = None,
241 ):
242 super().__init__()
243 self.post_layer_norm = config.post_layer_norm
@@ -246,7 +247,7 @@ class GLMTransformer(nn.Module):
247
248 # Transformer layers.
249 self.layers = nn.ModuleList(
249 - [GLMBlock(config, linear_method) for i in range(self.num_layers)])
250 + [GLMBlock(config, quant_config) for i in range(self.num_layers)])
251
252 if self.post_layer_norm:
253 layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -281,7 +282,7 @@ class ChatGLMModel(nn.Module):
282 def __init__(
283 self,
284 config,
284 - linear_method: Optional[LinearMethodBase] = None,
285 + quant_config: Optional[QuantizationConfig] = None,
286 ):
287 super().__init__()
288
@@ -291,7 +292,7 @@ class ChatGLMModel(nn.Module):
292 self.num_layers = config.num_layers
293 self.multi_query_group_num = config.multi_query_group_num
294 self.kv_channels = config.kv_channels
294 - self.encoder = GLMTransformer(config, linear_method)
295 + self.encoder = GLMTransformer(config, quant_config)
296
297 self.output_layer = ParallelLMHead(config.padded_vocab_size,
298 config.hidden_size)
@@ -333,13 +334,13 @@ class ChatGLMForCausalLM(nn.Module):
334 def __init__(
335 self,
336 config: ChatGLMConfig,
336 - linear_method: Optional[LinearMethodBase] = None,
337 + quant_config: Optional[QuantizationConfig] = None,
338 lora_config: Optional[LoRAConfig] = None,
339 ):
340 super().__init__()
341 self.config: ChatGLMConfig = config
341 - self.linear_method = linear_method
342 - self.transformer = ChatGLMModel(config, linear_method)
342 + self.quant_config = quant_config
343 + self.transformer = ChatGLMModel(config, quant_config)
344 self.lm_head_weight = self.transformer.output_layer.weight
345 self.logits_processor = LogitsProcessor(config.padded_vocab_size)
346 self.sampler = Sampler()
vllm/model_executor/models/commandr.py
+17 -16
@@ -32,11 +32,12 @@ from vllm.attention import Attention, AttentionMetadata
32 from vllm.distributed import (get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
35 -from vllm.model_executor.layers.linear import (LinearMethodBase,
36 - MergedColumnParallelLinear,
35 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
36 QKVParallelLinear,
37 RowParallelLinear)
38 from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 +from vllm.model_executor.layers.quantization.base_config import (
40 + QuantizationConfig)
41 from vllm.model_executor.layers.rotary_embedding import get_rope
42 from vllm.model_executor.layers.sampler import Sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -91,7 +92,7 @@ class CohereMLP(nn.Module):
92 def __init__(
93 self,
94 config,
94 - linear_method: Optional[LinearMethodBase] = None,
95 + quant_config: Optional[QuantizationConfig] = None,
96 ):
97 super().__init__()
98 self.config = config
@@ -101,13 +102,13 @@ class CohereMLP(nn.Module):
102 self.hidden_size,
103 [self.intermediate_size] * 2,
104 bias=False,
104 - linear_method=linear_method,
105 + quant_config=quant_config,
106 )
107 self.down_proj = RowParallelLinear(
108 self.intermediate_size,
109 self.hidden_size,
110 bias=False,
110 - linear_method=linear_method,
111 + quant_config=quant_config,
112 )
113 self.act_fn = SiluAndMul()
114
@@ -123,7 +124,7 @@ class CohereAttention(nn.Module):
124 def __init__(
125 self,
126 config: CohereConfig,
126 - linear_method: Optional[LinearMethodBase] = None,
127 + quant_config: Optional[QuantizationConfig] = None,
128 ):
129 super().__init__()
130 tp_size = get_tensor_model_parallel_world_size()
@@ -158,13 +159,13 @@ class CohereAttention(nn.Module):
159 self.total_num_heads,
160 self.total_num_kv_heads,
161 bias=False,
161 - linear_method=linear_method,
162 + quant_config=quant_config,
163 )
164 self.o_proj = RowParallelLinear(
165 self.total_num_heads * self.head_dim,
166 self.hidden_size,
167 bias=False,
167 - linear_method=linear_method,
168 + quant_config=quant_config,
169 )
170 self.rotary_emb = get_rope(
171 self.head_dim,
@@ -218,13 +219,13 @@ class CohereDecoderLayer(nn.Module):
219
220 def __init__(self,
221 config: CohereConfig,
221 - linear_method: Optional[LinearMethodBase] = None):
222 + quant_config: Optional[QuantizationConfig] = None):
223 super().__init__()
224 self.hidden_size = config.hidden_size
225
225 - self.self_attn = CohereAttention(config, linear_method=linear_method)
226 + self.self_attn = CohereAttention(config, quant_config=quant_config)
227
227 - self.mlp = CohereMLP(config, linear_method=linear_method)
228 + self.mlp = CohereMLP(config, quant_config=quant_config)
229 self.input_layernorm = LayerNorm(param_shape=(config.hidden_size),
230 eps=config.layer_norm_eps)
231
@@ -257,7 +258,7 @@ class CohereModel(nn.Module):
258 def __init__(
259 self,
260 config: CohereConfig,
260 - linear_method: Optional[LinearMethodBase] = None,
261 + quant_config: Optional[QuantizationConfig] = None,
262 ):
263 super().__init__()
264 self.config = config
@@ -265,7 +266,7 @@ class CohereModel(nn.Module):
266 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
267 config.hidden_size)
268 self.layers = nn.ModuleList([
268 - CohereDecoderLayer(config, linear_method=linear_method)
269 + CohereDecoderLayer(config, quant_config=quant_config)
270 for _ in range(config.num_hidden_layers)
271 ])
272 self.norm = LayerNorm(param_shape=(config.hidden_size),
@@ -298,14 +299,14 @@ class CohereForCausalLM(nn.Module):
299 def __init__(
300 self,
301 config: CohereConfig,
301 - linear_method: Optional[LinearMethodBase] = None,
302 + quant_config: Optional[QuantizationConfig] = None,
303 ) -> None:
304 super().__init__()
305 self.config = config
305 - self.linear_method = linear_method
306 + self.quant_config = quant_config
307 self.logits_processor = LogitsProcessor(config.vocab_size,
308 scale=config.logit_scale)
308 - self.model = CohereModel(config, linear_method)
309 + self.model = CohereModel(config, quant_config)
310 self.sampler = Sampler()
311
312 @torch.no_grad()
vllm/model_executor/models/dbrx.py
+18 -17
@@ -9,11 +9,12 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
9 get_tensor_model_parallel_world_size,
10 tensor_model_parallel_all_reduce)
11 from vllm.model_executor.layers.fused_moe import fused_moe
12 -from vllm.model_executor.layers.linear import (LinearMethodBase,
13 - QKVParallelLinear,
12 +from vllm.model_executor.layers.linear import (QKVParallelLinear,
13 ReplicatedLinear,
14 RowParallelLinear)
15 from vllm.model_executor.layers.logits_processor import LogitsProcessor
16 +from vllm.model_executor.layers.quantization.base_config import (
17 + QuantizationConfig)
18 from vllm.model_executor.layers.rotary_embedding import get_rope
19 from vllm.model_executor.layers.sampler import Sampler
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -44,7 +45,7 @@ class DbrxRouter(nn.Module):
45 self.num_total_experts,
46 bias=False,
47 params_dtype=params_dtype,
47 - linear_method=None,
48 + quant_config=None,
49 )
50
51 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -63,7 +64,7 @@ class DbrxExperts(nn.Module):
64 def __init__(
65 self,
66 config: DbrxConfig,
66 - linear_method: Optional[LinearMethodBase] = None,
67 + quant_config: Optional[QuantizationConfig] = None,
68 params_dtype: Optional[torch.dtype] = None,
69 ):
70 super().__init__()
@@ -165,7 +166,7 @@ class DbrxAttention(nn.Module):
166 def __init__(
167 self,
168 config: DbrxConfig,
168 - linear_method: Optional[LinearMethodBase] = None,
169 + quant_config: Optional[QuantizationConfig] = None,
170 ):
171 super().__init__()
172 self.d_model = config.d_model
@@ -183,13 +184,13 @@ class DbrxAttention(nn.Module):
184 self.total_num_heads,
185 self.total_num_kv_heads,
186 bias=False,
186 - linear_method=linear_method,
187 + quant_config=quant_config,
188 )
189 self.out_proj = RowParallelLinear(
190 self.d_model,
191 self.d_model,
192 bias=False,
192 - linear_method=linear_method,
193 + quant_config=quant_config,
194 )
195 self.rotary_emb = get_rope(
196 self.head_dim,
@@ -244,11 +245,11 @@ class DbrxFusedNormAttention(nn.Module):
245 def __init__(
246 self,
247 config: DbrxConfig,
247 - linear_method: Optional[LinearMethodBase] = None,
248 + quant_config: Optional[QuantizationConfig] = None,
249 ):
250 super().__init__()
251 self.d_model = config.d_model
251 - self.attn = DbrxAttention(config, linear_method)
252 + self.attn = DbrxAttention(config, quant_config)
253 self.norm_1 = nn.LayerNorm(self.d_model)
254 self.norm_2 = nn.LayerNorm(self.d_model)
255
@@ -278,11 +279,11 @@ class DbrxBlock(nn.Module):
279 def __init__(
280 self,
281 config: DbrxConfig,
281 - linear_method: Optional[LinearMethodBase] = None,
282 + quant_config: Optional[QuantizationConfig] = None,
283 ):
284 super().__init__()
284 - self.norm_attn_norm = DbrxFusedNormAttention(config, linear_method)
285 - self.ffn = DbrxExperts(config, linear_method)
285 + self.norm_attn_norm = DbrxFusedNormAttention(config, quant_config)
286 + self.ffn = DbrxExperts(config, quant_config)
287
288 def forward(
289 self,
@@ -307,7 +308,7 @@ class DbrxModel(nn.Module):
308 def __init__(
309 self,
310 config: DbrxConfig,
310 - linear_method: Optional[LinearMethodBase] = None,
311 + quant_config: Optional[QuantizationConfig] = None,
312 ):
313 super().__init__()
314 self.wte = VocabParallelEmbedding(
@@ -315,7 +316,7 @@ class DbrxModel(nn.Module):
316 config.d_model,
317 )
318 self.blocks = nn.ModuleList(
318 - [DbrxBlock(config, linear_method) for _ in range(config.n_layers)])
319 + [DbrxBlock(config, quant_config) for _ in range(config.n_layers)])
320 self.norm_f = nn.LayerNorm(config.d_model, eps=1e-5)
321 for module in self.modules():
322 if hasattr(module, "bias") and isinstance(module.bias,
@@ -348,13 +349,13 @@ class DbrxForCausalLM(nn.Module):
349 def __init__(
350 self,
351 config: DbrxConfig,
351 - linear_method: Optional[LinearMethodBase] = None,
352 + quant_config: Optional[QuantizationConfig] = None,
353 ):
354 super().__init__()
355 self.config = config
355 - self.linear_method = linear_method
356 + self.quant_config = quant_config
357 self.unpadded_vocab_size = config.vocab_size
357 - self.transformer = DbrxModel(config, linear_method)
358 + self.transformer = DbrxModel(config, quant_config)
359 self.lm_head = ParallelLMHead(
360 config.vocab_size,
361 config.d_model,
vllm/model_executor/models/decilm.py
+4 -3
@@ -29,7 +29,8 @@ import torch
29 from transformers import PretrainedConfig
30
31 from vllm.config import LoRAConfig
32 -from vllm.model_executor.layers.linear import LinearMethodBase
32 +from vllm.model_executor.layers.quantization.base_config import (
33 + QuantizationConfig)
34 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35 from vllm.model_executor.models.llama import LlamaForCausalLM
36
@@ -55,13 +56,13 @@ class DeciLMForCausalLM(LlamaForCausalLM):
56 def __init__(
57 self,
58 config: Optional[PretrainedConfig] = None,
58 - linear_method: Optional[LinearMethodBase] = None,
59 + quant_config: Optional[QuantizationConfig] = None,
60 lora_config: Optional[LoRAConfig] = None,
61 ) -> None:
62 config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
63 delattr(config, "num_key_value_heads_per_layer")
64 super().__init__(config=config,
64 - linear_method=linear_method,
65 + quant_config=quant_config,
66 lora_config=lora_config)
67
68 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
vllm/model_executor/models/deepseek.py
+22 -23
@@ -34,12 +34,13 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.fused_moe import fused_moe
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 -from vllm.model_executor.layers.linear import (LinearMethodBase,
38 - MergedColumnParallelLinear,
37 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
38 QKVParallelLinear,
39 ReplicatedLinear,
40 RowParallelLinear)
41 from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 +from vllm.model_executor.layers.quantization.base_config import (
43 + QuantizationConfig)
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 from vllm.model_executor.layers.sampler import Sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -56,18 +57,18 @@ class DeepseekMLP(nn.Module):
57 hidden_size: int,
58 intermediate_size: int,
59 hidden_act: str,
59 - linear_method: Optional[LinearMethodBase] = None,
60 + quant_config: Optional[QuantizationConfig] = None,
61 reduce_results: bool = True,
62 ) -> None:
63 super().__init__()
64 self.gate_up_proj = MergedColumnParallelLinear(
65 hidden_size, [intermediate_size] * 2,
66 bias=False,
66 - linear_method=linear_method)
67 + quant_config=quant_config)
68 self.down_proj = RowParallelLinear(intermediate_size,
69 hidden_size,
70 bias=False,
70 - linear_method=linear_method,
71 + quant_config=quant_config,
72 reduce_results=reduce_results)
73 if hidden_act != "silu":
74 raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -86,7 +87,7 @@ class DeepseekMoE(nn.Module):
87 def __init__(
88 self,
89 config: PretrainedConfig,
89 - linear_method: Optional[LinearMethodBase] = None,
90 + quant_config: Optional[QuantizationConfig] = None,
91 ):
92 super().__init__()
93 self.config = config
@@ -103,7 +104,7 @@ class DeepseekMoE(nn.Module):
104 DeepseekMLP(hidden_size=config.hidden_size,
105 intermediate_size=config.moe_intermediate_size,
106 hidden_act=config.hidden_act,
106 - linear_method=linear_method,
107 + quant_config=quant_config,
108 reduce_results=False)
109 for idx in range(self.n_routed_experts)
110 ])
@@ -112,7 +113,7 @@ class DeepseekMoE(nn.Module):
113 self.gate = ReplicatedLinear(config.hidden_size,
114 self.n_routed_experts,
115 bias=False,
115 - linear_method=None)
116 + quant_config=None)
117
118 if config.n_shared_experts is not None:
119 intermediate_size = (config.moe_intermediate_size *
@@ -121,7 +122,7 @@ class DeepseekMoE(nn.Module):
122 hidden_size=config.hidden_size,
123 intermediate_size=intermediate_size,
124 hidden_act=config.hidden_act,
124 - linear_method=linear_method,
125 + quant_config=quant_config,
126 reduce_results=False,
127 )
128
@@ -177,7 +178,7 @@ class DeepseekAttention(nn.Module):
178 rope_theta: float = 10000,
179 rope_scaling: Optional[Dict[str, Any]] = None,
180 max_position_embeddings: int = 8192,
180 - linear_method: Optional[LinearMethodBase] = None,
181 + quant_config: Optional[QuantizationConfig] = None,
182 ) -> None:
183 super().__init__()
184 self.hidden_size = hidden_size
@@ -208,14 +209,14 @@ class DeepseekAttention(nn.Module):
209 self.total_num_heads,
210 self.total_num_kv_heads,
211 bias=False,
211 - linear_method=linear_method,
212 + quant_config=quant_config,
213 )
214
215 self.o_proj = RowParallelLinear(
216 self.total_num_heads * self.head_dim,
217 hidden_size,
218 bias=False,
218 - linear_method=linear_method,
219 + quant_config=quant_config,
220 )
221
222 self.rotary_emb = get_rope(
@@ -251,7 +252,7 @@ class DeepseekDecoderLayer(nn.Module):
252 self,
253 config: PretrainedConfig,
254 layer_idx: int,
254 - linear_method: Optional[LinearMethodBase] = None,
255 + quant_config: Optional[QuantizationConfig] = None,
256 ) -> None:
257 super().__init__()
258 self.hidden_size = config.hidden_size
@@ -266,18 +267,18 @@ class DeepseekDecoderLayer(nn.Module):
267 rope_theta=rope_theta,
268 rope_scaling=rope_scaling,
269 max_position_embeddings=max_position_embeddings,
269 - linear_method=linear_method,
270 + quant_config=quant_config,
271 )
272 if (config.n_routed_experts is not None
273 and layer_idx >= config.first_k_dense_replace
274 and layer_idx % config.moe_layer_freq == 0):
274 - self.mlp = DeepseekMoE(config=config, linear_method=linear_method)
275 + self.mlp = DeepseekMoE(config=config, quant_config=quant_config)
276 else:
277 self.mlp = DeepseekMLP(
278 hidden_size=config.hidden_size,
279 intermediate_size=config.intermediate_size,
280 hidden_act=config.hidden_act,
280 - linear_method=linear_method,
281 + quant_config=quant_config,
282 )
283 self.input_layernorm = RMSNorm(config.hidden_size,
284 eps=config.rms_norm_eps)
@@ -320,7 +321,7 @@ class DeepseekModel(nn.Module):
321 def __init__(
322 self,
323 config: PretrainedConfig,
323 - linear_method: Optional[LinearMethodBase] = None,
324 + quant_config: Optional[QuantizationConfig] = None,
325 ) -> None:
326 super().__init__()
327 self.padding_idx = config.pad_token_id
@@ -331,9 +332,7 @@ class DeepseekModel(nn.Module):
332 config.hidden_size,
333 )
334 self.layers = nn.ModuleList([
334 - DeepseekDecoderLayer(config,
335 - layer_idx,
336 - linear_method=linear_method)
335 + DeepseekDecoderLayer(config, layer_idx, quant_config=quant_config)
336 for layer_idx in range(config.num_hidden_layers)
337 ])
338 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -361,12 +360,12 @@ class DeepseekForCausalLM(nn.Module):
360 def __init__(
361 self,
362 config: PretrainedConfig,
364 - linear_method: Optional[LinearMethodBase] = None,
363 + quant_config: Optional[QuantizationConfig] = None,
364 ) -> None:
365 super().__init__()
366 self.config = config
368 - self.linear_method = linear_method
369 - self.model = DeepseekModel(config, linear_method)
367 + self.quant_config = quant_config
368 + self.model = DeepseekModel(config, quant_config)
369 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
370 self.logits_processor = LogitsProcessor(config.vocab_size)
371 self.sampler = Sampler()
vllm/model_executor/models/falcon.py
+17 -16
@@ -32,10 +32,11 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
32 tensor_model_parallel_all_reduce)
33 from vllm.model_executor.layers.activation import get_act_fn
34 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35 - LinearMethodBase,
35 QKVParallelLinear,
36 RowParallelLinear)
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 +from vllm.model_executor.layers.quantization.base_config import (
39 + QuantizationConfig)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -76,7 +77,7 @@ class FalconAttention(nn.Module):
77 def __init__(
78 self,
79 config: FalconConfig,
79 - linear_method: Optional[LinearMethodBase] = None,
80 + quant_config: Optional[QuantizationConfig] = None,
81 ):
82 super().__init__()
83
@@ -115,7 +116,7 @@ class FalconAttention(nn.Module):
116 self.total_num_kv_heads,
117 bias=config.bias,
118 skip_bias_add=True,
118 - linear_method=linear_method,
119 + quant_config=quant_config,
120 )
121 self.q_size = self.num_heads * self.head_dim
122 self.kv_size = self.num_kv_heads * self.head_dim
@@ -129,7 +130,7 @@ class FalconAttention(nn.Module):
130 self.hidden_size,
131 bias=config.bias,
132 skip_bias_add=True,
132 - linear_method=linear_method,
133 + quant_config=quant_config,
134 reduce_results=self.reduce_row_parallel_results)
135
136 self.use_rotary = config.rotary
@@ -192,7 +193,7 @@ class FalconMLP(nn.Module):
193 def __init__(
194 self,
195 config: FalconConfig,
195 - linear_method: Optional[LinearMethodBase] = None,
196 + quant_config: Optional[QuantizationConfig] = None,
197 ):
198 super().__init__()
199 hidden_size = config.hidden_size
@@ -201,8 +202,8 @@ class FalconMLP(nn.Module):
202 4 * hidden_size,
203 bias=config.bias,
204 skip_bias_add=True,
204 - linear_method=linear_method)
205 - quant_config = getattr(linear_method, "quant_config", None)
205 + quant_config=quant_config)
206 + quant_config = getattr(quant_config, "quant_config", None)
207 self.act = get_act_fn("gelu", quant_config, 4 * hidden_size)
208 self.reduce_row_parallel_results = not (config.new_decoder_architecture
209 or config.parallel_attn)
@@ -212,7 +213,7 @@ class FalconMLP(nn.Module):
213 bias=config.bias,
214 skip_bias_add=True,
215 reduce_results=self.reduce_row_parallel_results,
215 - linear_method=linear_method)
216 + quant_config=quant_config)
217
218 def forward(self, x: torch.Tensor) -> torch.Tensor:
219 # NOTE(zhuohan): Following huggingface, we do not fuse bias add here.
@@ -229,13 +230,13 @@ class FalconDecoderLayer(nn.Module):
230 def __init__(
231 self,
232 config: FalconConfig,
232 - linear_method: Optional[LinearMethodBase] = None,
233 + quant_config: Optional[QuantizationConfig] = None,
234 ):
235 super().__init__()
236 hidden_size = config.hidden_size
237 self.num_heads = config.num_attention_heads
237 - self.self_attention = FalconAttention(config, linear_method)
238 - self.mlp = FalconMLP(config, linear_method)
238 + self.self_attention = FalconAttention(config, quant_config)
239 + self.mlp = FalconMLP(config, quant_config)
240 self.config = config
241
242 if config.new_decoder_architecture:
@@ -311,7 +312,7 @@ class FalconModel(nn.Module):
312 def __init__(
313 self,
314 config: FalconConfig,
314 - linear_method: Optional[LinearMethodBase] = None,
315 + quant_config: Optional[QuantizationConfig] = None,
316 ):
317 super().__init__()
318 self.config = config
@@ -327,7 +328,7 @@ class FalconModel(nn.Module):
328
329 # Transformer blocks
330 self.h = nn.ModuleList([
330 - FalconDecoderLayer(config, linear_method)
331 + FalconDecoderLayer(config, quant_config)
332 for _ in range(config.num_hidden_layers)
333 ])
334
@@ -359,12 +360,12 @@ class FalconForCausalLM(nn.Module):
360 def __init__(
361 self,
362 config: FalconConfig,
362 - linear_method: Optional[LinearMethodBase] = None,
363 + quant_config: Optional[QuantizationConfig] = None,
364 ):
365 super().__init__()
366 self.config = config
366 - self.linear_method = linear_method
367 - self.transformer = FalconModel(config, linear_method)
367 + self.quant_config = quant_config
368 + self.transformer = FalconModel(config, quant_config)
369 self.lm_head_weight = self.transformer.word_embeddings.weight
370 self.logits_processor = LogitsProcessor(config.vocab_size)
371 self.sampler = Sampler()
vllm/model_executor/models/gemma.py
+17 -16
@@ -27,11 +27,12 @@ from vllm.distributed import get_tensor_model_parallel_world_size
27 from vllm.logger import init_logger
28 from vllm.model_executor.layers.activation import GeluAndMul
29 from vllm.model_executor.layers.layernorm import RMSNorm
30 -from vllm.model_executor.layers.linear import (LinearMethodBase,
31 - MergedColumnParallelLinear,
30 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31 QKVParallelLinear,
32 RowParallelLinear)
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 +from vllm.model_executor.layers.quantization.base_config import (
35 + QuantizationConfig)
36 from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -77,17 +78,17 @@ class GemmaMLP(nn.Module):
78 intermediate_size: int,
79 hidden_act: Optional[str] = None,
80 hidden_activation: Optional[str] = None,
80 - linear_method: Optional[LinearMethodBase] = None,
81 + quant_config: Optional[QuantizationConfig] = None,
82 ) -> None:
83 super().__init__()
84 self.gate_up_proj = MergedColumnParallelLinear(
85 hidden_size, [intermediate_size] * 2,
86 bias=False,
86 - linear_method=linear_method)
87 + quant_config=quant_config)
88 self.down_proj = RowParallelLinear(intermediate_size,
89 hidden_size,
90 bias=False,
90 - linear_method=linear_method)
91 + quant_config=quant_config)
92 self.act_fn = _get_gemma_act_fn(hidden_act, hidden_activation)
93
94 def forward(self, x):
@@ -106,7 +107,7 @@ class GemmaAttention(nn.Module):
107 head_dim: int,
108 max_position_embeddings: int = 8192,
109 rope_theta: float = 10000,
109 - linear_method: Optional[LinearMethodBase] = None) -> None:
110 + quant_config: Optional[QuantizationConfig] = None) -> None:
111 super().__init__()
112 self.hidden_size = hidden_size
113 tp_size = get_tensor_model_parallel_world_size()
@@ -135,13 +136,13 @@ class GemmaAttention(nn.Module):
136 self.total_num_heads,
137 self.total_num_kv_heads,
138 bias=False,
138 - linear_method=linear_method,
139 + quant_config=quant_config,
140 )
141 self.o_proj = RowParallelLinear(
142 self.total_num_heads * self.head_dim,
143 hidden_size,
144 bias=False,
144 - linear_method=linear_method,
145 + quant_config=quant_config,
146 )
147
148 self.rotary_emb = get_rope(
@@ -176,7 +177,7 @@ class GemmaDecoderLayer(nn.Module):
177 def __init__(
178 self,
179 config: GemmaConfig,
179 - linear_method: Optional[LinearMethodBase] = None,
180 + quant_config: Optional[QuantizationConfig] = None,
181 ) -> None:
182 super().__init__()
183 self.hidden_size = config.hidden_size
@@ -187,14 +188,14 @@ class GemmaDecoderLayer(nn.Module):
188 head_dim=config.head_dim,
189 max_position_embeddings=config.max_position_embeddings,
190 rope_theta=config.rope_theta,
190 - linear_method=linear_method,
191 + quant_config=quant_config,
192 )
193 self.mlp = GemmaMLP(
194 hidden_size=self.hidden_size,
195 intermediate_size=config.intermediate_size,
196 hidden_act=config.hidden_act,
197 hidden_activation=getattr(config, "hidden_activation", None),
197 - linear_method=linear_method,
198 + quant_config=quant_config,
199 )
200 self.input_layernorm = RMSNorm(config.hidden_size,
201 eps=config.rms_norm_eps)
@@ -235,7 +236,7 @@ class GemmaModel(nn.Module):
236 def __init__(
237 self,
238 config: GemmaConfig,
238 - linear_method: Optional[LinearMethodBase] = None,
239 + quant_config: Optional[QuantizationConfig] = None,
240 ) -> None:
241 super().__init__()
242 self.config = config
@@ -245,7 +246,7 @@ class GemmaModel(nn.Module):
246 config.hidden_size,
247 )
248 self.layers = nn.ModuleList([
248 - GemmaDecoderLayer(config, linear_method)
249 + GemmaDecoderLayer(config, quant_config)
250 for _ in range(config.num_hidden_layers)
251 ])
252 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -308,14 +309,14 @@ class GemmaForCausalLM(nn.Module):
309 def __init__(
310 self,
311 config: GemmaConfig,
311 - linear_method: Optional[LinearMethodBase] = None,
312 + quant_config: Optional[QuantizationConfig] = None,
313 lora_config: Optional[LoRAConfig] = None,
314 ) -> None:
315 del lora_config # Unused.
316 super().__init__()
317 self.config = config
317 - self.linear_method = linear_method
318 - self.model = GemmaModel(config, linear_method)
318 + self.quant_config = quant_config
319 + self.model = GemmaModel(config, quant_config)
320 self.logits_processor = LogitsProcessor(config.vocab_size)
321 self.sampler = Sampler()
322
vllm/model_executor/models/gpt2.py
+17 -16
@@ -27,10 +27,11 @@ from vllm.attention import Attention, AttentionMetadata
27 from vllm.distributed import get_tensor_model_parallel_world_size
28 from vllm.model_executor.layers.activation import get_act_fn
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 - LinearMethodBase,
30 QKVParallelLinear,
31 RowParallelLinear)
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 +from vllm.model_executor.layers.quantization.base_config import (
34 + QuantizationConfig)
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
@@ -44,7 +45,7 @@ class GPT2Attention(nn.Module):
45 def __init__(
46 self,
47 config: GPT2Config,
47 - linear_method: Optional[LinearMethodBase] = None,
48 + quant_config: Optional[QuantizationConfig] = None,
49 ):
50 super().__init__()
51 self.hidden_size = config.hidden_size
@@ -61,13 +62,13 @@ class GPT2Attention(nn.Module):
62 self.head_dim,
63 total_num_heads,
64 bias=True,
64 - linear_method=linear_method,
65 + quant_config=quant_config,
66 )
67 self.c_proj = RowParallelLinear(
68 self.hidden_size,
69 self.hidden_size,
70 bias=True,
70 - linear_method=linear_method,
71 + quant_config=quant_config,
72 )
73 self.attn = Attention(self.num_heads, self.head_dim, scale=self.scale)
74
@@ -90,7 +91,7 @@ class GPT2MLP(nn.Module):
91 self,
92 intermediate_size: int,
93 config: GPT2Config,
93 - linear_method: Optional[LinearMethodBase] = None,
94 + quant_config: Optional[QuantizationConfig] = None,
95 ):
96 super().__init__()
97 hidden_size = config.hidden_size
@@ -98,15 +99,15 @@ class GPT2MLP(nn.Module):
99 hidden_size,
100 intermediate_size,
101 bias=True,
101 - linear_method=linear_method,
102 + quant_config=quant_config,
103 )
104 self.c_proj = RowParallelLinear(
105 intermediate_size,
106 hidden_size,
107 bias=True,
107 - linear_method=linear_method,
108 + quant_config=quant_config,
109 )
109 - quant_config = getattr(linear_method, "quant_config", None)
110 + quant_config = getattr(quant_config, "quant_config", None)
111 self.act = get_act_fn(config.activation_function, quant_config,
112 intermediate_size)
113
@@ -122,7 +123,7 @@ class GPT2Block(nn.Module):
123 def __init__(
124 self,
125 config: GPT2Config,
125 - linear_method: Optional[LinearMethodBase] = None,
126 + quant_config: Optional[QuantizationConfig] = None,
127 ):
128 super().__init__()
129 hidden_size = config.hidden_size
@@ -130,9 +131,9 @@ class GPT2Block(nn.Module):
131 hidden_size)
132
133 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
133 - self.attn = GPT2Attention(config, linear_method)
134 + self.attn = GPT2Attention(config, quant_config)
135 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
135 - self.mlp = GPT2MLP(inner_dim, config, linear_method)
136 + self.mlp = GPT2MLP(inner_dim, config, quant_config)
137
138 def forward(
139 self,
@@ -163,7 +164,7 @@ class GPT2Model(nn.Module):
164 def __init__(
165 self,
166 config: GPT2Config,
166 - linear_method: Optional[LinearMethodBase] = None,
167 + quant_config: Optional[QuantizationConfig] = None,
168 ):
169 super().__init__()
170 self.config = config
@@ -174,7 +175,7 @@ class GPT2Model(nn.Module):
175 self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
176 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
177 self.h = nn.ModuleList([
177 - GPT2Block(config, linear_method)
178 + GPT2Block(config, quant_config)
179 for _ in range(config.num_hidden_layers)
180 ])
181 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -203,12 +204,12 @@ class GPT2LMHeadModel(nn.Module):
204 def __init__(
205 self,
206 config: GPT2Config,
206 - linear_method: Optional[LinearMethodBase] = None,
207 + quant_config: Optional[QuantizationConfig] = None,
208 ):
209 super().__init__()
210 self.config = config
210 - self.linear_method = linear_method
211 - self.transformer = GPT2Model(config, linear_method)
211 + self.quant_config = quant_config
212 + self.transformer = GPT2Model(config, quant_config)
213 self.lm_head_weight = self.transformer.wte.weight
214 self.logits_processor = LogitsProcessor(config.vocab_size)
215 self.sampler = Sampler()
vllm/model_executor/models/gpt_bigcode.py
+17 -16
@@ -28,10 +28,11 @@ from vllm.attention import Attention, AttentionMetadata
28 from vllm.distributed import get_tensor_model_parallel_world_size
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 - LinearMethodBase,
31 QKVParallelLinear,
32 RowParallelLinear)
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 +from vllm.model_executor.layers.quantization.base_config import (
35 + QuantizationConfig)
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
@@ -45,7 +46,7 @@ class GPTBigCodeAttention(nn.Module):
46 def __init__(
47 self,
48 config: GPTBigCodeConfig,
48 - linear_method: Optional[LinearMethodBase] = None,
49 + quant_config: Optional[QuantizationConfig] = None,
50 ):
51 super().__init__()
52 self.hidden_size = config.hidden_size
@@ -72,14 +73,14 @@ class GPTBigCodeAttention(nn.Module):
73 total_num_heads,
74 total_num_kv_heads,
75 bias=True,
75 - linear_method=linear_method,
76 + quant_config=quant_config,
77 )
78
79 self.c_proj = RowParallelLinear(
80 self.hidden_size,
81 self.hidden_size,
82 bias=True,
82 - linear_method=linear_method,
83 + quant_config=quant_config,
84 )
85 self.attn = Attention(self.num_heads,
86 self.head_dim,
@@ -111,7 +112,7 @@ class GPTBigMLP(nn.Module):
112 self,
113 intermediate_size: int,
114 config: GPTBigCodeConfig,
114 - linear_method: Optional[LinearMethodBase] = None,
115 + quant_config: Optional[QuantizationConfig] = None,
116 ):
117 super().__init__()
118 hidden_size = config.hidden_size
@@ -119,15 +120,15 @@ class GPTBigMLP(nn.Module):
120 hidden_size,
121 intermediate_size,
122 bias=True,
122 - linear_method=linear_method,
123 + quant_config=quant_config,
124 )
125 self.c_proj = RowParallelLinear(
126 intermediate_size,
127 hidden_size,
128 bias=True,
128 - linear_method=linear_method,
129 + quant_config=quant_config,
130 )
130 - quant_config = getattr(linear_method, "quant_config", None)
131 + quant_config = getattr(quant_config, "quant_config", None)
132 self.act = get_act_fn(config.activation_function, quant_config,
133 intermediate_size)
134
@@ -143,7 +144,7 @@ class GPTBigCodeBlock(nn.Module):
144 def __init__(
145 self,
146 config: GPTBigCodeConfig,
146 - linear_method: Optional[LinearMethodBase] = None,
147 + quant_config: Optional[QuantizationConfig] = None,
148 ):
149 super().__init__()
150 hidden_size = config.hidden_size
@@ -151,9 +152,9 @@ class GPTBigCodeBlock(nn.Module):
152 hidden_size)
153
154 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
154 - self.attn = GPTBigCodeAttention(config, linear_method)
155 + self.attn = GPTBigCodeAttention(config, quant_config)
156 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
156 - self.mlp = GPTBigMLP(inner_dim, config, linear_method)
157 + self.mlp = GPTBigMLP(inner_dim, config, quant_config)
158
159 def forward(
160 self,
@@ -184,7 +185,7 @@ class GPTBigCodeModel(nn.Module):
185 def __init__(
186 self,
187 config: GPTBigCodeConfig,
187 - linear_method: Optional[LinearMethodBase] = None,
188 + quant_config: Optional[QuantizationConfig] = None,
189 ):
190 super().__init__()
191 self.config = config
@@ -195,7 +196,7 @@ class GPTBigCodeModel(nn.Module):
196 self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
197 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
198 self.h = nn.ModuleList([
198 - GPTBigCodeBlock(config, linear_method)
199 + GPTBigCodeBlock(config, quant_config)
200 for _ in range(config.num_hidden_layers)
201 ])
202 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -224,12 +225,12 @@ class GPTBigCodeForCausalLM(nn.Module):
225 def __init__(
226 self,
227 config: GPTBigCodeConfig,
227 - linear_method: Optional[LinearMethodBase] = None,
228 + quant_config: Optional[QuantizationConfig] = None,
229 ):
230 super().__init__()
231 self.config = config
231 - self.linear_method = linear_method
232 - self.transformer = GPTBigCodeModel(config, linear_method)
232 + self.quant_config = quant_config
233 + self.transformer = GPTBigCodeModel(config, quant_config)
234 self.lm_head_weight = self.transformer.wte.weight
235 self.logits_processor = LogitsProcessor(config.vocab_size)
236 self.sampler = Sampler()
vllm/model_executor/models/gpt_j.py
+17 -16
@@ -26,10 +26,11 @@ from vllm.attention import Attention, AttentionMetadata
26 from vllm.distributed import get_tensor_model_parallel_world_size
27 from vllm.model_executor.layers.activation import get_act_fn
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 - LinearMethodBase,
29 QKVParallelLinear,
30 RowParallelLinear)
31 from vllm.model_executor.layers.logits_processor import LogitsProcessor
32 +from vllm.model_executor.layers.quantization.base_config import (
33 + QuantizationConfig)
34 from vllm.model_executor.layers.rotary_embedding import get_rope
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -44,7 +45,7 @@ class GPTJAttention(nn.Module):
45 def __init__(
46 self,
47 config: GPTJConfig,
47 - linear_method: Optional[LinearMethodBase] = None,
48 + quant_config: Optional[QuantizationConfig] = None,
49 ):
50 super().__init__()
51 self.total_num_heads = config.num_attention_heads
@@ -56,13 +57,13 @@ class GPTJAttention(nn.Module):
57 self.head_size,
58 self.total_num_heads,
59 bias=False,
59 - linear_method=linear_method,
60 + quant_config=quant_config,
61 )
62 self.out_proj = RowParallelLinear(
63 config.hidden_size,
64 config.hidden_size,
65 bias=False,
65 - linear_method=linear_method,
66 + quant_config=quant_config,
67 )
68
69 tp_world_size = get_tensor_model_parallel_world_size()
@@ -105,21 +106,21 @@ class GPTJMLP(nn.Module):
106 self,
107 intermediate_size: int,
108 config: GPTJConfig,
108 - linear_method: Optional[LinearMethodBase] = None,
109 + quant_config: Optional[QuantizationConfig] = None,
110 ):
111 super().__init__()
112 hidden_size = config.n_embd
113 self.fc_in = ColumnParallelLinear(
114 hidden_size,
115 intermediate_size,
115 - linear_method=linear_method,
116 + quant_config=quant_config,
117 )
118 self.fc_out = RowParallelLinear(
119 intermediate_size,
120 hidden_size,
120 - linear_method=linear_method,
121 + quant_config=quant_config,
122 )
122 - quant_config = getattr(linear_method, "quant_config", None)
123 + quant_config = getattr(quant_config, "quant_config", None)
124 self.act = get_act_fn(config.activation_function, quant_config,
125 intermediate_size)
126
@@ -135,14 +136,14 @@ class GPTJBlock(nn.Module):
136 def __init__(
137 self,
138 config: GPTJConfig,
138 - linear_method: Optional[LinearMethodBase] = None,
139 + quant_config: Optional[QuantizationConfig] = None,
140 ):
141 super().__init__()
142 inner_dim = (4 * config.n_embd
143 if config.n_inner is None else config.n_inner)
144 self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
144 - self.attn = GPTJAttention(config, linear_method)
145 - self.mlp = GPTJMLP(inner_dim, config, linear_method)
145 + self.attn = GPTJAttention(config, quant_config)
146 + self.mlp = GPTJMLP(inner_dim, config, quant_config)
147
148 def forward(
149 self,
@@ -169,7 +170,7 @@ class GPTJModel(nn.Module):
170 def __init__(
171 self,
172 config: GPTJConfig,
172 - linear_method: Optional[LinearMethodBase] = None,
173 + quant_config: Optional[QuantizationConfig] = None,
174 ):
175 super().__init__()
176 self.config = config
@@ -179,7 +180,7 @@ class GPTJModel(nn.Module):
180 self.embed_dim,
181 )
182 self.h = nn.ModuleList(
182 - [GPTJBlock(config, linear_method) for _ in range(config.n_layer)])
183 + [GPTJBlock(config, quant_config) for _ in range(config.n_layer)])
184 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
185
186 def forward(
@@ -207,13 +208,13 @@ class GPTJForCausalLM(nn.Module):
208 def __init__(
209 self,
210 config: GPTJConfig,
210 - linear_method: Optional[LinearMethodBase] = None,
211 + quant_config: Optional[QuantizationConfig] = None,
212 ):
213 super().__init__()
214 self.config = config
214 - self.linear_method = linear_method
215 + self.quant_config = quant_config
216 assert not config.tie_word_embeddings
216 - self.transformer = GPTJModel(config, linear_method)
217 + self.transformer = GPTJModel(config, quant_config)
218 self.lm_head = ParallelLMHead(
219 config.vocab_size,
220 config.n_embd,
vllm/model_executor/models/gpt_neox.py
+17 -16
@@ -26,10 +26,11 @@ from vllm.attention import Attention, AttentionMetadata
26 from vllm.distributed import get_tensor_model_parallel_world_size
27 from vllm.model_executor.layers.activation import get_act_fn
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 - LinearMethodBase,
29 QKVParallelLinear,
30 RowParallelLinear)
31 from vllm.model_executor.layers.logits_processor import LogitsProcessor
32 +from vllm.model_executor.layers.quantization.base_config import (
33 + QuantizationConfig)
34 from vllm.model_executor.layers.rotary_embedding import get_rope
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -44,7 +45,7 @@ class GPTNeoXAttention(nn.Module):
45 def __init__(
46 self,
47 config: GPTNeoXConfig,
47 - linear_method: Optional[LinearMethodBase] = None,
48 + quant_config: Optional[QuantizationConfig] = None,
49 ):
50 super().__init__()
51 self.total_num_heads = config.num_attention_heads
@@ -63,13 +64,13 @@ class GPTNeoXAttention(nn.Module):
64 self.head_size,
65 self.total_num_heads,
66 bias=self.bias,
66 - linear_method=linear_method,
67 + quant_config=quant_config,
68 )
69 self.dense = RowParallelLinear(
70 config.hidden_size,
71 config.hidden_size,
72 bias=self.bias,
72 - linear_method=linear_method,
73 + quant_config=quant_config,
74 )
75 scaling = self.head_size**-0.5
76 rotary_dim = int(self.head_size * config.rotary_pct)
@@ -105,20 +106,20 @@ class GPTNeoXMLP(nn.Module):
106 def __init__(
107 self,
108 config: GPTNeoXConfig,
108 - linear_method: Optional[LinearMethodBase] = None,
109 + quant_config: Optional[QuantizationConfig] = None,
110 ):
111 super().__init__()
112 self.dense_h_to_4h = ColumnParallelLinear(
113 config.hidden_size,
114 config.intermediate_size,
114 - linear_method=linear_method,
115 + quant_config=quant_config,
116 )
117 self.dense_4h_to_h = RowParallelLinear(
118 config.intermediate_size,
119 config.hidden_size,
119 - linear_method=linear_method,
120 + quant_config=quant_config,
121 )
121 - quant_config = getattr(linear_method, "quant_config", None)
122 + quant_config = getattr(quant_config, "quant_config", None)
123 self.act = get_act_fn(config.hidden_act, quant_config,
124 config.intermediate_size)
125
@@ -134,7 +135,7 @@ class GPTNeoXLayer(nn.Module):
135 def __init__(
136 self,
137 config: GPTNeoXConfig,
137 - linear_method: Optional[LinearMethodBase] = None,
138 + quant_config: Optional[QuantizationConfig] = None,
139 ):
140 super().__init__()
141 self.use_parallel_residual = config.use_parallel_residual
@@ -142,8 +143,8 @@ class GPTNeoXLayer(nn.Module):
143 eps=config.layer_norm_eps)
144 self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
145 eps=config.layer_norm_eps)
145 - self.attention = GPTNeoXAttention(config, linear_method)
146 - self.mlp = GPTNeoXMLP(config, linear_method)
146 + self.attention = GPTNeoXAttention(config, quant_config)
147 + self.mlp = GPTNeoXMLP(config, quant_config)
148
149 def forward(
150 self,
@@ -182,7 +183,7 @@ class GPTNeoXModel(nn.Module):
183 def __init__(
184 self,
185 config: GPTNeoXConfig,
185 - linear_method: Optional[LinearMethodBase] = None,
186 + quant_config: Optional[QuantizationConfig] = None,
187 ):
188 super().__init__()
189 self.config = config
@@ -192,7 +193,7 @@ class GPTNeoXModel(nn.Module):
193 config.hidden_size,
194 )
195 self.layers = nn.ModuleList([
195 - GPTNeoXLayer(config, linear_method)
196 + GPTNeoXLayer(config, quant_config)
197 for _ in range(config.num_hidden_layers)
198 ])
199 self.final_layer_norm = nn.LayerNorm(config.hidden_size,
@@ -223,12 +224,12 @@ class GPTNeoXForCausalLM(nn.Module):
224 def __init__(
225 self,
226 config,
226 - linear_method: Optional[LinearMethodBase] = None,
227 + quant_config: Optional[QuantizationConfig] = None,
228 ):
229 super().__init__()
230 self.config = config
230 - self.linear_method = linear_method
231 - self.gpt_neox = GPTNeoXModel(config, linear_method)
231 + self.quant_config = quant_config
232 + self.gpt_neox = GPTNeoXModel(config, quant_config)
233 self.embed_out = ParallelLMHead(
234 config.vocab_size,
235 config.hidden_size,
vllm/model_executor/models/internlm2.py
+17 -16
@@ -9,11 +9,12 @@ from vllm.attention import Attention, AttentionMetadata
9 from vllm.distributed import get_tensor_model_parallel_world_size
10 from vllm.model_executor.layers.activation import SiluAndMul
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 -from vllm.model_executor.layers.linear import (LinearMethodBase,
13 - MergedColumnParallelLinear,
12 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
13 QKVParallelLinear,
14 RowParallelLinear)
15 from vllm.model_executor.layers.logits_processor import LogitsProcessor
16 +from vllm.model_executor.layers.quantization.base_config import (
17 + QuantizationConfig)
18 from vllm.model_executor.layers.rotary_embedding import get_rope
19 from vllm.model_executor.layers.sampler import Sampler
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -30,17 +31,17 @@ class InternLM2MLP(nn.Module):
31 hidden_size: int,
32 intermediate_size: int,
33 hidden_act: str,
33 - linear_method: Optional[LinearMethodBase] = None,
34 + quant_config: Optional[QuantizationConfig] = None,
35 ) -> None:
36 super().__init__()
37 self.gate_up_proj = MergedColumnParallelLinear(
38 hidden_size, [intermediate_size] * 2,
39 bias=False,
39 - linear_method=linear_method)
40 + quant_config=quant_config)
41 self.w2 = RowParallelLinear(intermediate_size,
42 hidden_size,
43 bias=False,
43 - linear_method=linear_method)
44 + quant_config=quant_config)
45 if hidden_act != "silu":
46 raise ValueError(f"Unsupported activation: {hidden_act}. "
47 "Only silu is supported for now.")
@@ -63,7 +64,7 @@ class InternLM2Attention(nn.Module):
64 rope_theta: float = 10000,
65 rope_scaling: Optional[Dict[str, Any]] = None,
66 max_position_embeddings: int = 8192,
66 - linear_method: Optional[LinearMethodBase] = None,
67 + quant_config: Optional[QuantizationConfig] = None,
68 ) -> None:
69 super().__init__()
70 self.hidden_size = hidden_size
@@ -94,13 +95,13 @@ class InternLM2Attention(nn.Module):
95 self.total_num_heads,
96 self.total_num_kv_heads,
97 bias=False,
97 - linear_method=linear_method,
98 + quant_config=quant_config,
99 )
100 self.wo = RowParallelLinear(
101 self.total_num_heads * self.head_dim,
102 hidden_size,
103 bias=False,
103 - linear_method=linear_method,
104 + quant_config=quant_config,
105 )
106
107 self.rotary_emb = get_rope(
@@ -135,7 +136,7 @@ class InternLMDecoderLayer(nn.Module):
136 def __init__(
137 self,
138 config: PretrainedConfig,
138 - linear_method: Optional[LinearMethodBase] = None,
139 + quant_config: Optional[QuantizationConfig] = None,
140 ) -> None:
141 super().__init__()
142 self.hidden_size = config.hidden_size
@@ -150,13 +151,13 @@ class InternLMDecoderLayer(nn.Module):
151 rope_theta=rope_theta,
152 rope_scaling=rope_scaling,
153 max_position_embeddings=max_position_embeddings,
153 - linear_method=linear_method,
154 + quant_config=quant_config,
155 )
156 self.feed_forward = InternLM2MLP(
157 hidden_size=self.hidden_size,
158 intermediate_size=config.intermediate_size,
159 hidden_act=config.hidden_act,
159 - linear_method=linear_method,
160 + quant_config=quant_config,
161 )
162 self.attention_norm = RMSNorm(config.hidden_size,
163 eps=config.rms_norm_eps)
@@ -195,7 +196,7 @@ class InternLM2Model(nn.Module):
196 def __init__(
197 self,
198 config: PretrainedConfig,
198 - linear_method: Optional[LinearMethodBase] = None,
199 + quant_config: Optional[QuantizationConfig] = None,
200 ) -> None:
201 super().__init__()
202 self.config = config
@@ -206,7 +207,7 @@ class InternLM2Model(nn.Module):
207 config.hidden_size,
208 )
209 self.layers = nn.ModuleList([
209 - InternLMDecoderLayer(config, linear_method)
210 + InternLMDecoderLayer(config, quant_config)
211 for _ in range(config.num_hidden_layers)
212 ])
213 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -238,12 +239,12 @@ class InternLM2ForCausalLM(nn.Module):
239 def __init__(
240 self,
241 config: PretrainedConfig,
241 - linear_method: Optional[LinearMethodBase] = None,
242 + quant_config: Optional[QuantizationConfig] = None,
243 ) -> None:
244 super().__init__()
245 self.config = config
245 - self.linear_method = linear_method
246 - self.model = InternLM2Model(config, linear_method)
246 + self.quant_config = quant_config
247 + self.model = InternLM2Model(config, quant_config)
248 self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
249 self.logits_processor = LogitsProcessor(config.vocab_size)
250 self.sampler = Sampler()
vllm/model_executor/models/jais.py
+17 -16
@@ -29,10 +29,11 @@ from vllm.attention import Attention, AttentionMetadata
29 from vllm.distributed import (get_tensor_model_parallel_rank,
30 get_tensor_model_parallel_world_size)
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 - LinearMethodBase,
32 QKVParallelLinear,
33 RowParallelLinear)
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 +from vllm.model_executor.layers.quantization.base_config import (
36 + QuantizationConfig)
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
@@ -68,7 +69,7 @@ class JAISAttention(nn.Module):
69 def __init__(
70 self,
71 config: JAISConfig,
71 - linear_method: Optional[LinearMethodBase] = None,
72 + quant_config: Optional[QuantizationConfig] = None,
73 ):
74 super().__init__()
75 self.hidden_size = config.hidden_size
@@ -88,13 +89,13 @@ class JAISAttention(nn.Module):
89 self.head_dim,
90 total_num_heads,
91 bias=True,
91 - linear_method=linear_method,
92 + quant_config=quant_config,
93 )
94 self.c_proj = RowParallelLinear(
95 self.hidden_size,
96 self.hidden_size,
97 bias=True,
97 - linear_method=linear_method,
98 + quant_config=quant_config,
99 )
100
101 tp_rank = get_tensor_model_parallel_rank()
@@ -128,7 +129,7 @@ class JAISMLP(nn.Module):
129 self,
130 intermediate_size: int,
131 config: JAISConfig,
131 - linear_method: Optional[LinearMethodBase] = None,
132 + quant_config: Optional[QuantizationConfig] = None,
133 ):
134 super().__init__()
135 hidden_size = config.hidden_size
@@ -137,19 +138,19 @@ class JAISMLP(nn.Module):
138 hidden_size,
139 intermediate_size,
140 bias=True,
140 - linear_method=linear_method,
141 + quant_config=quant_config,
142 )
143 self.c_fc2 = (ColumnParallelLinear(
144 hidden_size,
145 intermediate_size,
146 bias=True,
146 - linear_method=linear_method,
147 + quant_config=quant_config,
148 ) if self.swiglu else None)
149 self.c_proj = RowParallelLinear(
150 intermediate_size,
151 hidden_size,
152 bias=True,
152 - linear_method=linear_method,
153 + quant_config=quant_config,
154 )
155
156 self.act = SwiGLUActivation()
@@ -169,7 +170,7 @@ class JAISBlock(nn.Module):
170 def __init__(
171 self,
172 config: JAISConfig,
172 - linear_method: Optional[LinearMethodBase] = None,
173 + quant_config: Optional[QuantizationConfig] = None,
174 ):
175 super().__init__()
176 hidden_size = config.hidden_size
@@ -177,9 +178,9 @@ class JAISBlock(nn.Module):
178 hidden_size)
179
180 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
180 - self.attn = JAISAttention(config, linear_method)
181 + self.attn = JAISAttention(config, quant_config)
182 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
182 - self.mlp = JAISMLP(inner_dim, config, linear_method)
183 + self.mlp = JAISMLP(inner_dim, config, quant_config)
184
185 def forward(
186 self,
@@ -210,7 +211,7 @@ class JAISModel(nn.Module):
211 def __init__(
212 self,
213 config: JAISConfig,
213 - linear_method: Optional[LinearMethodBase] = None,
214 + quant_config: Optional[QuantizationConfig] = None,
215 ):
216 super().__init__()
217 self.config = config
@@ -227,7 +228,7 @@ class JAISModel(nn.Module):
228 else:
229 self.embeddings_scale = config.mup_embeddings_scale
230 self.h = nn.ModuleList([
230 - JAISBlock(config, linear_method)
231 + JAISBlock(config, quant_config)
232 for _ in range(config.num_hidden_layers)
233 ])
234 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -261,12 +262,12 @@ class JAISLMHeadModel(nn.Module):
262 def __init__(
263 self,
264 config: JAISConfig,
264 - linear_method: Optional[LinearMethodBase] = None,
265 + quant_config: Optional[QuantizationConfig] = None,
266 ):
267 super().__init__()
268 self.config = config
268 - self.linear_method = linear_method
269 - self.transformer = JAISModel(config, linear_method)
269 + self.quant_config = quant_config
270 + self.transformer = JAISModel(config, quant_config)
271 self.lm_head_weight = self.transformer.wte.weight
272 if hasattr(config, "width_scale"):
273 self.output_logits_scale = config.width_scale
vllm/model_executor/models/llama.py
+16 -16
@@ -33,11 +33,12 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
36 -from vllm.model_executor.layers.linear import (LinearMethodBase,
37 - MergedColumnParallelLinear,
36 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37 QKVParallelLinear,
38 RowParallelLinear)
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 +from vllm.model_executor.layers.quantization.base_config import (
41 + QuantizationConfig)
42 from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import Sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -56,17 +57,17 @@ class LlamaMLP(nn.Module):
57 hidden_size: int,
58 intermediate_size: int,
59 hidden_act: str,
59 - linear_method: Optional[LinearMethodBase] = None,
60 + quant_config: Optional[QKVParallelLinear] = None,
61 ) -> None:
62 super().__init__()
63 self.gate_up_proj = MergedColumnParallelLinear(
64 hidden_size, [intermediate_size] * 2,
65 bias=False,
65 - linear_method=linear_method)
66 + quant_config=quant_config)
67 self.down_proj = RowParallelLinear(intermediate_size,
68 hidden_size,
69 bias=False,
69 - linear_method=linear_method)
70 + quant_config=quant_config)
71 if hidden_act != "silu":
72 raise ValueError(f"Unsupported activation: {hidden_act}. "
73 "Only silu is supported for now.")
@@ -89,7 +90,7 @@ class LlamaAttention(nn.Module):
90 rope_theta: float = 10000,
91 rope_scaling: Optional[Dict[str, Any]] = None,
92 max_position_embeddings: int = 8192,
92 - linear_method: Optional[LinearMethodBase] = None,
93 + quant_config: Optional[QuantizationConfig] = None,
94 bias: bool = False,
95 sliding_window: Optional[int] = None,
96 ) -> None:
@@ -131,13 +132,13 @@ class LlamaAttention(nn.Module):
132 self.total_num_heads,
133 self.total_num_kv_heads,
134 bias=bias,
134 - linear_method=linear_method,
135 + quant_config=quant_config,
136 )
137 self.o_proj = RowParallelLinear(
138 self.total_num_heads * self.head_dim,
139 hidden_size,
140 bias=bias,
140 - linear_method=linear_method,
141 + quant_config=quant_config,
142 )
143
144 self.rotary_emb = get_rope(
@@ -174,7 +175,7 @@ class LlamaDecoderLayer(nn.Module):
175 def __init__(
176 self,
177 config: LlamaConfig,
177 - linear_method: Optional[LinearMethodBase] = None,
178 + quant_config: Optional[QuantizationConfig] = None,
179 ) -> None:
180 super().__init__()
181 self.hidden_size = config.hidden_size
@@ -199,7 +200,7 @@ class LlamaDecoderLayer(nn.Module):
200 rope_theta=rope_theta,
201 rope_scaling=rope_scaling,
202 max_position_embeddings=max_position_embeddings,
202 - linear_method=linear_method,
203 + quant_config=quant_config,
204 bias=attention_bias,
205 sliding_window=sliding_window,
206 )
@@ -207,7 +208,7 @@ class LlamaDecoderLayer(nn.Module):
208 hidden_size=self.hidden_size,
209 intermediate_size=config.intermediate_size,
210 hidden_act=config.hidden_act,
210 - linear_method=linear_method,
211 + quant_config=quant_config,
212 )
213 self.input_layernorm = RMSNorm(config.hidden_size,
214 eps=config.rms_norm_eps)
@@ -248,7 +249,7 @@ class LlamaModel(nn.Module):
249 def __init__(
250 self,
251 config: LlamaConfig,
251 - linear_method: Optional[LinearMethodBase] = None,
252 + quant_config: Optional[QuantizationConfig] = None,
253 lora_config: Optional[LoRAConfig] = None,
254 ) -> None:
255 super().__init__()
@@ -264,7 +265,7 @@ class LlamaModel(nn.Module):
265 org_num_embeddings=config.vocab_size,
266 )
267 self.layers = nn.ModuleList([
267 - LlamaDecoderLayer(config, linear_method)
268 + LlamaDecoderLayer(config, quant_config)
269 for _ in range(config.num_hidden_layers)
270 ])
271 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -329,13 +330,12 @@ class LlamaForCausalLM(nn.Module):
330 def __init__(
331 self,
332 config: LlamaConfig,
332 - linear_method: Optional[LinearMethodBase] = None,
333 + quant_config: Optional[QuantizationConfig] = None,
334 lora_config: Optional[LoRAConfig] = None,
335 ) -> None:
336 super().__init__()
337 self.config = config
337 - self.linear_method = linear_method
338 - self.model = LlamaModel(config, linear_method, lora_config=lora_config)
338 + self.model = LlamaModel(config, quant_config, lora_config=lora_config)
339 self.unpadded_vocab_size = config.vocab_size
340 if lora_config:
341 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/llava.py
+5 -4
@@ -9,8 +9,9 @@ from transformers import CLIPVisionModel, LlavaConfig
9 from vllm.attention import AttentionMetadata
10 from vllm.config import VisionLanguageConfig
11 from vllm.model_executor.layers.activation import get_act_fn
12 -from vllm.model_executor.layers.linear import LinearMethodBase
12 from vllm.model_executor.layers.logits_processor import LogitsProcessor
13 +from vllm.model_executor.layers.quantization.base_config import (
14 + QuantizationConfig)
15 from vllm.model_executor.layers.sampler import Sampler
16 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
17 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -61,7 +62,7 @@ class LlavaForConditionalGeneration(nn.Module):
62 def __init__(self,
63 config: "LlavaConfig",
64 vision_language_config: VisionLanguageConfig,
64 - linear_method: Optional["LinearMethodBase"] = None) -> None:
65 + quant_config: Optional["QuantizationConfig"] = None) -> None:
66 super().__init__()
67 self.config = config
68
@@ -83,8 +84,8 @@ class LlavaForConditionalGeneration(nn.Module):
84 text_hidden_size=config.text_config.hidden_size,
85 projector_hidden_act=config.projector_hidden_act)
86
86 - self.linear_method = linear_method
87 - self.language_model = LlamaModel(config.text_config, linear_method)
87 + self.quant_config = quant_config
88 + self.language_model = LlamaModel(config.text_config, quant_config)
89 self.unpadded_vocab_size = config.text_config.vocab_size
90 self.lm_head = ParallelLMHead(
91 self.unpadded_vocab_size,
vllm/model_executor/models/minicpm.py
+18 -17
@@ -35,12 +35,13 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
35 from vllm.model_executor.layers.activation import SiluAndMul
36 from vllm.model_executor.layers.fused_moe import fused_moe
37 from vllm.model_executor.layers.layernorm import RMSNorm
38 -from vllm.model_executor.layers.linear import (LinearMethodBase,
39 - MergedColumnParallelLinear,
38 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39 QKVParallelLinear,
40 ReplicatedLinear,
41 RowParallelLinear)
42 from vllm.model_executor.layers.logits_processor import LogitsProcessor
43 +from vllm.model_executor.layers.quantization.base_config import (
44 + QuantizationConfig)
45 from vllm.model_executor.layers.rotary_embedding import get_rope
46 from vllm.model_executor.layers.sampler import Sampler
47 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -84,7 +85,7 @@ class MiniCPMMoE(nn.Module):
85 self.num_total_experts,
86 bias=False,
87 params_dtype=self.params_dtype,
87 - linear_method=None)
88 + quant_config=None)
89
90 self.ws = nn.Parameter(
91 torch.empty(self.num_total_experts,
@@ -147,17 +148,17 @@ class MiniCPMMLP(nn.Module):
148 hidden_size: int,
149 intermediate_size: int,
150 hidden_act: str,
150 - linear_method: Optional[LinearMethodBase] = None,
151 + quant_config: Optional[QuantizationConfig] = None,
152 ) -> None:
153 super().__init__()
154 self.gate_up_proj = MergedColumnParallelLinear(
155 hidden_size, [intermediate_size] * 2,
156 bias=False,
156 - linear_method=linear_method)
157 + quant_config=quant_config)
158 self.down_proj = RowParallelLinear(intermediate_size,
159 hidden_size,
160 bias=False,
160 - linear_method=linear_method)
161 + quant_config=quant_config)
162 if hidden_act != "silu":
163 raise ValueError(f"Unsupported activation: {hidden_act}. "
164 "Only silu is supported for now.")
@@ -180,7 +181,7 @@ class MiniCPMAttention(nn.Module):
181 rope_theta: float = 10000,
182 rope_scaling: Optional[Dict[str, Any]] = None,
183 max_position_embeddings: int = 8192,
183 - linear_method: Optional[LinearMethodBase] = None,
184 + quant_config: Optional[QuantizationConfig] = None,
185 ) -> None:
186 super().__init__()
187 self.hidden_size = hidden_size
@@ -211,13 +212,13 @@ class MiniCPMAttention(nn.Module):
212 self.total_num_heads,
213 self.total_num_kv_heads,
214 bias=False,
214 - linear_method=linear_method,
215 + quant_config=quant_config,
216 )
217 self.o_proj = RowParallelLinear(
218 self.total_num_heads * self.head_dim,
219 hidden_size,
220 bias=False,
220 - linear_method=linear_method,
221 + quant_config=quant_config,
222 )
223
224 self.rotary_emb = get_rope(
@@ -258,7 +259,7 @@ class MiniCPMDecoderLayer(nn.Module):
259 def __init__(
260 self,
261 config,
261 - linear_method: Optional[LinearMethodBase] = None,
262 + quant_config: Optional[QuantizationConfig] = None,
263 ) -> None:
264 super().__init__()
265 self.config = config
@@ -274,7 +275,7 @@ class MiniCPMDecoderLayer(nn.Module):
275 rope_theta=rope_theta,
276 rope_scaling=rope_scaling,
277 max_position_embeddings=max_position_embeddings,
277 - linear_method=linear_method,
278 + quant_config=quant_config,
279 )
280 self.num_experts = getattr(self.config, "num_experts", 0)
281 if self.num_experts == 0:
@@ -282,7 +283,7 @@ class MiniCPMDecoderLayer(nn.Module):
283 hidden_size=self.hidden_size,
284 intermediate_size=config.intermediate_size,
285 hidden_act=config.hidden_act,
285 - linear_method=linear_method,
286 + quant_config=quant_config,
287 )
288 else:
289 self.mlp = MiniCPMMoE(num_experts=config.num_experts,
@@ -329,7 +330,7 @@ class MiniCPMModel(nn.Module):
330 def __init__(
331 self,
332 config,
332 - linear_method: Optional[LinearMethodBase] = None,
333 + quant_config: Optional[QuantizationConfig] = None,
334 lora_config: Optional[LoRAConfig] = None,
335 ) -> None:
336 super().__init__()
@@ -345,7 +346,7 @@ class MiniCPMModel(nn.Module):
346 org_num_embeddings=config.vocab_size,
347 )
348 self.layers = nn.ModuleList([
348 - MiniCPMDecoderLayer(config, linear_method)
349 + MiniCPMDecoderLayer(config, quant_config)
350 for _ in range(config.num_hidden_layers)
351 ])
352 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -412,15 +413,15 @@ class MiniCPMForCausalLM(nn.Module):
413 def __init__(
414 self,
415 config,
415 - linear_method: Optional[LinearMethodBase] = None,
416 + quant_config: Optional[QuantizationConfig] = None,
417 lora_config: Optional[LoRAConfig] = None,
418 ) -> None:
419 super().__init__()
420 self.config = config
421 self.num_experts = getattr(self.config, "num_experts", 0)
421 - self.linear_method = linear_method
422 + self.quant_config = quant_config
423 self.model = MiniCPMModel(config,
423 - linear_method,
424 + quant_config,
425 lora_config=lora_config)
426 unpadded_vocab_size = config.vocab_size
427 if lora_config:
vllm/model_executor/models/mixtral.py
+22 -22
@@ -27,6 +27,7 @@ import torch
27 from torch import nn
28 from transformers import MixtralConfig
29
30 +from vllm import _custom_ops as ops
31 from vllm.attention import Attention, AttentionMetadata
32 from vllm.config import LoRAConfig
33 from vllm.distributed import (get_tensor_model_parallel_rank,
@@ -34,13 +35,13 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
35 tensor_model_parallel_all_reduce)
36 from vllm.model_executor.layers.fused_moe import fused_moe
37 from vllm.model_executor.layers.layernorm import RMSNorm
37 -from vllm.model_executor.layers.linear import (LinearMethodBase,
38 - QKVParallelLinear,
38 +from vllm.model_executor.layers.linear import (QKVParallelLinear,
39 ReplicatedLinear,
40 RowParallelLinear)
41 from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 -from vllm.model_executor.layers.quantization.fp8 import (Fp8LinearMethod,
43 - per_tensor_quantize)
42 +from vllm.model_executor.layers.quantization.base_config import (
43 + QuantizationConfig)
44 +from vllm.model_executor.layers.quantization.fp8 import Fp8Config
45 from vllm.model_executor.layers.rotary_embedding import get_rope
46 from vllm.model_executor.layers.sampler import Sampler
47 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -69,7 +70,7 @@ class MixtralMoE(nn.Module):
70 intermediate_size: int,
71 params_dtype: Optional[torch.dtype] = None,
72 tp_size: Optional[int] = None,
72 - linear_method: Optional[LinearMethodBase] = None,
73 + quant_config: Optional[QuantizationConfig] = None,
74 ):
75 super().__init__()
76 self.tp_size = tp_size or get_tensor_model_parallel_world_size()
@@ -79,7 +80,7 @@ class MixtralMoE(nn.Module):
80 self.intermediate_size = intermediate_size // self.tp_size
81 # FIXME(pcmoritz): Make this more general to support different
82 # quantization schemes
82 - self.use_fp8 = isinstance(linear_method, Fp8LinearMethod)
83 + self.use_fp8 = isinstance(quant_config, Fp8Config)
84
85 if params_dtype is None:
86 params_dtype = torch.get_default_dtype()
@@ -89,7 +90,7 @@ class MixtralMoE(nn.Module):
90 self.num_total_experts,
91 bias=False,
92 params_dtype=self.params_dtype,
92 - linear_method=None)
93 + quant_config=None)
94
95 self.ws = nn.Parameter(
96 torch.empty(self.num_total_experts,
@@ -140,10 +141,10 @@ class MixtralMoE(nn.Module):
141 ws = torch.empty_like(self.ws.data, dtype=torch.float8_e4m3fn)
142 w2s = torch.empty_like(self.w2s.data, dtype=torch.float8_e4m3fn)
143 for expert in range(self.num_total_experts):
143 - ws[expert, :, :], self.ws_scale[expert] = per_tensor_quantize(
144 + ws[expert, :, :], self.ws_scale[expert] = ops.scaled_fp8_quant(
145 self.ws.data[expert, :, :])
146 w2s[expert, :, :], self.w2s_scale[
146 - expert] = per_tensor_quantize(self.w2s.data[expert, :, :])
147 + expert] = ops.scaled_fp8_quant(self.w2s.data[expert, :, :])
148 self.ws = nn.Parameter(ws, requires_grad=False)
149 self.w2s = nn.Parameter(w2s, requires_grad=False)
150
@@ -178,7 +179,7 @@ class MixtralAttention(nn.Module):
179 num_kv_heads: int,
180 max_position: int = 4096 * 32,
181 rope_theta: float = 10000,
181 - linear_method: Optional[LinearMethodBase] = None,
182 + quant_config: Optional[QuantizationConfig] = None,
183 sliding_window: Optional[int] = None) -> None:
184 super().__init__()
185 self.hidden_size = hidden_size
@@ -203,12 +204,12 @@ class MixtralAttention(nn.Module):
204 self.rope_theta = rope_theta
205 self.sliding_window = sliding_window
206
206 - if isinstance(linear_method, Fp8LinearMethod):
207 + if isinstance(quant_config, Fp8Config):
208 print_warning_once(
209 "For Mixtral FP8 quantization, we currently do not quantize "
210 "the attention layers until their FP8 performance is improved."
211 )
211 - linear_method = None
212 + quant_config = None
213
214 self.qkv_proj = QKVParallelLinear(
215 hidden_size,
@@ -216,13 +217,13 @@ class MixtralAttention(nn.Module):
217 self.total_num_heads,
218 self.total_num_kv_heads,
219 bias=False,
219 - linear_method=linear_method,
220 + quant_config=quant_config,
221 )
222 self.o_proj = RowParallelLinear(
223 self.total_num_heads * self.head_dim,
224 hidden_size,
225 bias=False,
225 - linear_method=linear_method,
226 + quant_config=quant_config,
227 )
228 self.rotary_emb = get_rope(
229 self.head_dim,
@@ -259,7 +260,7 @@ class MixtralDecoderLayer(nn.Module):
260 def __init__(
261 self,
262 config: MixtralConfig,
262 - linear_method: Optional[LinearMethodBase] = None,
263 + quant_config: Optional[QuantizationConfig] = None,
264 ) -> None:
265 super().__init__()
266 self.hidden_size = config.hidden_size
@@ -272,13 +273,13 @@ class MixtralDecoderLayer(nn.Module):
273 num_kv_heads=config.num_key_value_heads,
274 rope_theta=rope_theta,
275 sliding_window=config.sliding_window,
275 - linear_method=linear_method)
276 + quant_config=quant_config)
277 self.block_sparse_moe = MixtralMoE(
278 num_experts=config.num_local_experts,
279 top_k=config.num_experts_per_tok,
280 hidden_size=config.hidden_size,
281 intermediate_size=config.intermediate_size,
281 - linear_method=linear_method)
282 + quant_config=quant_config)
283 self.input_layernorm = RMSNorm(config.hidden_size,
284 eps=config.rms_norm_eps)
285 self.post_attention_layernorm = RMSNorm(config.hidden_size,
@@ -318,7 +319,7 @@ class MixtralModel(nn.Module):
319 def __init__(
320 self,
321 config: MixtralConfig,
321 - linear_method: Optional[LinearMethodBase] = None,
322 + quant_config: Optional[QuantizationConfig] = None,
323 lora_config: Optional[LoRAConfig] = None,
324 ) -> None:
325 super().__init__()
@@ -334,7 +335,7 @@ class MixtralModel(nn.Module):
335 org_num_embeddings=config.vocab_size,
336 )
337 self.layers = nn.ModuleList([
337 - MixtralDecoderLayer(config, linear_method=linear_method)
338 + MixtralDecoderLayer(config, quant_config=quant_config)
339 for _ in range(config.num_hidden_layers)
340 ])
341 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -384,14 +385,13 @@ class MixtralForCausalLM(nn.Module):
385 def __init__(
386 self,
387 config: MixtralConfig,
387 - linear_method: Optional[LinearMethodBase] = None,
388 + quant_config: Optional[QuantizationConfig] = None,
389 lora_config: Optional[LoRAConfig] = None,
390 ) -> None:
391 super().__init__()
392 self.config = config
392 - self.linear_method = linear_method
393 self.model = MixtralModel(config,
394 - linear_method,
394 + quant_config,
395 lora_config=lora_config)
396 self.unpadded_vocab_size = config.vocab_size
397 if lora_config:
vllm/model_executor/models/mixtral_quant.py
+21 -20
@@ -34,11 +34,12 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size,
35 tensor_model_parallel_all_reduce)
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 -from vllm.model_executor.layers.linear import (LinearMethodBase,
38 - QKVParallelLinear,
37 +from vllm.model_executor.layers.linear import (QKVParallelLinear,
38 ReplicatedLinear,
39 RowParallelLinear)
40 from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 +from vllm.model_executor.layers.quantization.base_config import (
42 + QuantizationConfig)
43 from vllm.model_executor.layers.rotary_embedding import get_rope
44 from vllm.model_executor.layers.sampler import Sampler
45 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -55,7 +56,7 @@ class MixtralMLP(nn.Module):
56 num_experts: int,
57 hidden_size: int,
58 intermediate_size: int,
58 - linear_method: Optional[LinearMethodBase] = None,
59 + quant_config: Optional[QuantizationConfig] = None,
60 ) -> None:
61 super().__init__()
62 self.num_experts = num_experts
@@ -65,15 +66,15 @@ class MixtralMLP(nn.Module):
66 self.w1 = ReplicatedLinear(self.hidden_dim,
67 self.ffn_dim,
68 bias=False,
68 - linear_method=linear_method)
69 + quant_config=quant_config)
70 self.w2 = ReplicatedLinear(self.ffn_dim,
71 self.hidden_dim,
72 bias=False,
72 - linear_method=linear_method)
73 + quant_config=quant_config)
74 self.w3 = ReplicatedLinear(self.hidden_dim,
75 self.ffn_dim,
76 bias=False,
76 - linear_method=linear_method)
77 + quant_config=quant_config)
78
79 # TODO: Use vllm's SiluAndMul
80 self.act_fn = nn.SiLU()
@@ -92,7 +93,7 @@ class MixtralMoE(nn.Module):
93 def __init__(
94 self,
95 config: MixtralConfig,
95 - linear_method: Optional[LinearMethodBase] = None,
96 + quant_config: Optional[QuantizationConfig] = None,
97 ):
98 super().__init__()
99 self.config = config
@@ -115,14 +116,14 @@ class MixtralMoE(nn.Module):
116 MixtralMLP(self.num_total_experts,
117 config.hidden_size,
118 config.intermediate_size,
118 - linear_method=linear_method)
119 + quant_config=quant_config)
120 if idx in self.expert_indicies else None
121 for idx in range(self.num_total_experts)
122 ])
123 self.gate = ReplicatedLinear(config.hidden_size,
124 self.num_total_experts,
125 bias=False,
125 - linear_method=None)
126 + quant_config=None)
127
128 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
129 num_tokens, hidden_dim = hidden_states.shape
@@ -162,7 +163,7 @@ class MixtralAttention(nn.Module):
163 num_kv_heads: int,
164 max_position: int = 4096 * 32,
165 rope_theta: float = 10000,
165 - linear_method: Optional[LinearMethodBase] = None,
166 + quant_config: Optional[QuantizationConfig] = None,
167 sliding_window: Optional[int] = None) -> None:
168 super().__init__()
169 self.hidden_size = hidden_size
@@ -193,13 +194,13 @@ class MixtralAttention(nn.Module):
194 self.total_num_heads,
195 self.total_num_kv_heads,
196 bias=False,
196 - linear_method=linear_method,
197 + quant_config=quant_config,
198 )
199 self.o_proj = RowParallelLinear(
200 self.total_num_heads * self.head_dim,
201 hidden_size,
202 bias=False,
202 - linear_method=linear_method,
203 + quant_config=quant_config,
204 )
205 self.rotary_emb = get_rope(
206 self.head_dim,
@@ -236,7 +237,7 @@ class MixtralDecoderLayer(nn.Module):
237 def __init__(
238 self,
239 config: MixtralConfig,
239 - linear_method: Optional[LinearMethodBase] = None,
240 + quant_config: Optional[QuantizationConfig] = None,
241 ) -> None:
242 super().__init__()
243 self.hidden_size = config.hidden_size
@@ -249,9 +250,9 @@ class MixtralDecoderLayer(nn.Module):
250 num_kv_heads=config.num_key_value_heads,
251 rope_theta=rope_theta,
252 sliding_window=config.sliding_window,
252 - linear_method=linear_method)
253 + quant_config=quant_config)
254 self.block_sparse_moe = MixtralMoE(config=config,
254 - linear_method=linear_method)
255 + quant_config=quant_config)
256 self.input_layernorm = RMSNorm(config.hidden_size,
257 eps=config.rms_norm_eps)
258 self.post_attention_layernorm = RMSNorm(config.hidden_size,
@@ -291,7 +292,7 @@ class MixtralModel(nn.Module):
292 def __init__(
293 self,
294 config: MixtralConfig,
294 - linear_method: Optional[LinearMethodBase] = None,
295 + quant_config: Optional[QuantizationConfig] = None,
296 ) -> None:
297 super().__init__()
298 self.padding_idx = config.pad_token_id
@@ -302,7 +303,7 @@ class MixtralModel(nn.Module):
303 config.hidden_size,
304 )
305 self.layers = nn.ModuleList([
305 - MixtralDecoderLayer(config, linear_method=linear_method)
306 + MixtralDecoderLayer(config, quant_config=quant_config)
307 for _ in range(config.num_hidden_layers)
308 ])
309 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -331,12 +332,12 @@ class MixtralForCausalLM(nn.Module):
332 def __init__(
333 self,
334 config: MixtralConfig,
334 - linear_method: Optional[LinearMethodBase] = None,
335 + quant_config: Optional[QuantizationConfig] = None,
336 ) -> None:
337 super().__init__()
338 self.config = config
338 - self.linear_method = linear_method
339 - self.model = MixtralModel(config, linear_method)
339 + self.quant_config = quant_config
340 + self.model = MixtralModel(config, quant_config)
341 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
342 self.logits_processor = LogitsProcessor(config.vocab_size)
343 self.sampler = Sampler()
vllm/model_executor/models/mpt.py
+17 -16
@@ -11,10 +11,11 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
11 get_tensor_model_parallel_world_size)
12 from vllm.model_executor.layers.activation import get_act_fn
13 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 - LinearMethodBase,
14 QKVParallelLinear,
15 RowParallelLinear)
16 from vllm.model_executor.layers.logits_processor import LogitsProcessor
17 +from vllm.model_executor.layers.quantization.base_config import (
18 + QuantizationConfig)
19 from vllm.model_executor.layers.sampler import Sampler
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
21 VocabParallelEmbedding)
@@ -42,7 +43,7 @@ class MPTAttention(nn.Module):
43 def __init__(
44 self,
45 config: MPTConfig,
45 - linear_method: Optional[LinearMethodBase] = None,
46 + quant_config: Optional[QuantizationConfig] = None,
47 ):
48 super().__init__()
49 self.d_model = config.d_model
@@ -65,7 +66,7 @@ class MPTAttention(nn.Module):
66 self.total_num_heads,
67 self.total_num_kv_heads,
68 bias=not config.no_bias,
68 - linear_method=linear_method,
69 + quant_config=quant_config,
70 )
71 if self.qk_ln:
72 self.q_ln = nn.LayerNorm(self.d_model)
@@ -74,7 +75,7 @@ class MPTAttention(nn.Module):
75 self.d_model,
76 self.d_model,
77 bias=not config.no_bias,
77 - linear_method=linear_method,
78 + quant_config=quant_config,
79 )
80
81 tp_world_size = get_tensor_model_parallel_world_size()
@@ -133,7 +134,7 @@ class MPTMLP(nn.Module):
134 def __init__(
135 self,
136 config: MPTConfig,
136 - linear_method: Optional[LinearMethodBase] = None,
137 + quant_config: Optional[QuantizationConfig] = None,
138 ):
139 super().__init__()
140 hidden_size = config.d_model
@@ -143,15 +144,15 @@ class MPTMLP(nn.Module):
144 hidden_size,
145 intermediate_size,
146 bias=not config.no_bias,
146 - linear_method=linear_method,
147 + quant_config=quant_config,
148 )
148 - quant_config = getattr(linear_method, "quant_config", None)
149 + quant_config = getattr(quant_config, "quant_config", None)
150 self.act = get_act_fn("gelu", quant_config, intermediate_size)
151 self.down_proj = RowParallelLinear(
152 intermediate_size,
153 hidden_size,
154 bias=not config.no_bias,
154 - linear_method=linear_method,
155 + quant_config=quant_config,
156 )
157
158 def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -166,14 +167,14 @@ class MPTBlock(nn.Module):
167 def __init__(
168 self,
169 config: MPTConfig,
169 - linear_method: Optional[LinearMethodBase] = None,
170 + quant_config: Optional[QuantizationConfig] = None,
171 ):
172 super().__init__()
173 hidden_size = config.d_model
174 self.norm_1 = nn.LayerNorm(hidden_size)
174 - self.attn = MPTAttention(config, linear_method)
175 + self.attn = MPTAttention(config, quant_config)
176 self.norm_2 = nn.LayerNorm(hidden_size)
176 - self.ffn = MPTMLP(config, linear_method)
177 + self.ffn = MPTMLP(config, quant_config)
178
179 def forward(
180 self,
@@ -201,7 +202,7 @@ class MPTModel(nn.Module):
202 def __init__(
203 self,
204 config: MPTConfig,
204 - linear_method: Optional[LinearMethodBase] = None,
205 + quant_config: Optional[QuantizationConfig] = None,
206 ):
207 super().__init__()
208 assert config.embedding_fraction == 1.0
@@ -212,7 +213,7 @@ class MPTModel(nn.Module):
213 config.d_model,
214 )
215 self.blocks = nn.ModuleList(
215 - [MPTBlock(config, linear_method) for _ in range(config.n_layers)])
216 + [MPTBlock(config, quant_config) for _ in range(config.n_layers)])
217 self.norm_f = nn.LayerNorm(config.d_model)
218 if config.no_bias:
219 for module in self.modules():
@@ -246,14 +247,14 @@ class MPTForCausalLM(nn.Module):
247 def __init__(
248 self,
249 config: MPTConfig,
249 - linear_method: Optional[LinearMethodBase] = None,
250 + quant_config: Optional[QuantizationConfig] = None,
251 ):
252 super().__init__()
253 self.config = config
254 assert config.tie_word_embeddings
254 - self.linear_method = linear_method
255 + self.quant_config = quant_config
256
256 - self.transformer = MPTModel(config, linear_method)
257 + self.transformer = MPTModel(config, quant_config)
258 self.lm_head_weight = self.transformer.wte.weight
259 self.logits_processor = LogitsProcessor(config.vocab_size)
260 self.sampler = Sampler()
vllm/model_executor/models/olmo.py
+16 -16
@@ -30,11 +30,12 @@ from transformers import OlmoConfig
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.distributed import get_tensor_model_parallel_world_size
32 from vllm.model_executor.layers.activation import SiluAndMul
33 -from vllm.model_executor.layers.linear import (LinearMethodBase,
34 - MergedColumnParallelLinear,
33 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
34 QKVParallelLinear,
35 RowParallelLinear)
36 from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 +from vllm.model_executor.layers.quantization.base_config import (
38 + QuantizationConfig)
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -54,7 +55,7 @@ class OlmoAttention(nn.Module):
55 def __init__(
56 self,
57 config: OlmoConfig,
57 - linear_method: Optional[LinearMethodBase] = None,
58 + quant_config: Optional[QuantizationConfig] = None,
59 ):
60 super().__init__()
61 self.config = config
@@ -79,7 +80,7 @@ class OlmoAttention(nn.Module):
80 self.head_dim,
81 self.total_num_heads,
82 bias=config.attention_bias,
82 - linear_method=linear_method,
83 + quant_config=quant_config,
84 )
85
86 # Rotary embeddings.
@@ -99,7 +100,7 @@ class OlmoAttention(nn.Module):
100 self.hidden_size,
101 self.hidden_size,
102 bias=config.attention_bias,
102 - linear_method=linear_method,
103 + quant_config=quant_config,
104 )
105
106 def forward(
@@ -129,7 +130,7 @@ class OlmoMLP(nn.Module):
130 def __init__(
131 self,
132 config: OlmoConfig,
132 - linear_method: Optional[LinearMethodBase] = None,
133 + quant_config: Optional[QuantizationConfig] = None,
134 ):
135 super().__init__()
136 self.config = config
@@ -141,7 +142,7 @@ class OlmoMLP(nn.Module):
142 self.hidden_size,
143 [self.intermediate_size] * 2,
144 bias=False,
144 - linear_method=linear_method,
145 + quant_config=quant_config,
146 )
147
148 # Activation function.
@@ -152,7 +153,7 @@ class OlmoMLP(nn.Module):
153 self.intermediate_size,
154 self.hidden_size,
155 bias=False,
155 - linear_method=linear_method,
156 + quant_config=quant_config,
157 )
158
159 def forward(
@@ -174,13 +175,13 @@ class OlmoDecoderLayer(nn.Module):
175
176 def __init__(self,
177 config: OlmoConfig,
177 - linear_method: Optional[LinearMethodBase] = None):
178 + quant_config: Optional[QuantizationConfig] = None):
179 super().__init__()
180 # Attention block.
180 - self.self_attn = OlmoAttention(config, linear_method)
181 + self.self_attn = OlmoAttention(config, quant_config)
182
183 # MLP block.
183 - self.mlp = OlmoMLP(config, linear_method)
184 + self.mlp = OlmoMLP(config, quant_config)
185
186 # LayerNorm
187 self.input_layernorm = nn.LayerNorm(config.hidden_size,
@@ -216,14 +217,14 @@ class OlmoModel(nn.Module):
217
218 def __init__(self,
219 config: OlmoConfig,
219 - linear_method: Optional[LinearMethodBase] = None):
220 + quant_config: Optional[QuantizationConfig] = None):
221 super().__init__()
222 self.config = config
223
224 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
225 config.hidden_size)
226 self.layers = nn.ModuleList([
226 - OlmoDecoderLayer(config, linear_method)
227 + OlmoDecoderLayer(config, quant_config)
228 for layer_idx in range(config.num_hidden_layers)
229 ])
230 self.norm = nn.LayerNorm(config.hidden_size,
@@ -270,11 +271,10 @@ class OlmoForCausalLM(nn.Module):
271
272 def __init__(self,
273 config: OlmoConfig,
273 - linear_method: Optional[LinearMethodBase] = None):
274 + quant_config: Optional[QuantizationConfig] = None):
275 super().__init__()
276 self.config = config
276 - self.linear_method = linear_method
277 - self.model = OlmoModel(config, linear_method)
277 + self.model = OlmoModel(config, quant_config)
278 if config.tie_word_embeddings:
279 self.lm_head_weight = self.model.embed_tokens.weight
280 else:
vllm/model_executor/models/opt.py
+19 -18
@@ -27,11 +27,12 @@ from vllm.attention import Attention, AttentionMetadata
27 from vllm.distributed import get_tensor_model_parallel_world_size
28 from vllm.model_executor.layers.activation import get_act_fn
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 - LinearMethodBase,
30 QKVParallelLinear,
31 ReplicatedLinear,
32 RowParallelLinear)
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 +from vllm.model_executor.layers.quantization.base_config import (
35 + QuantizationConfig)
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
@@ -60,7 +61,7 @@ class OPTAttention(nn.Module):
61 embed_dim: int,
62 num_heads: int,
63 bias: bool = True,
63 - linear_method: Optional[LinearMethodBase] = None,
64 + quant_config: Optional[QuantizationConfig] = None,
65 ) -> None:
66 super().__init__()
67 self.embed_dim = embed_dim
@@ -77,13 +78,13 @@ class OPTAttention(nn.Module):
78 self.head_dim,
79 total_num_heads,
80 bias=bias,
80 - linear_method=linear_method,
81 + quant_config=quant_config,
82 )
83 self.out_proj = RowParallelLinear(
84 embed_dim,
85 embed_dim,
86 bias=bias,
86 - linear_method=linear_method,
87 + quant_config=quant_config,
88 )
89 self.attn = Attention(self.num_heads,
90 self.head_dim,
@@ -107,7 +108,7 @@ class OPTDecoderLayer(nn.Module):
108 def __init__(
109 self,
110 config: OPTConfig,
110 - linear_method: Optional[LinearMethodBase] = None,
111 + quant_config: Optional[QuantizationConfig] = None,
112 ):
113 super().__init__()
114 self.config = config
@@ -116,7 +117,7 @@ class OPTDecoderLayer(nn.Module):
117 embed_dim=self.embed_dim,
118 num_heads=config.num_attention_heads,
119 bias=config.enable_bias,
119 - linear_method=linear_method,
120 + quant_config=quant_config,
121 )
122 self.do_layer_norm_before = config.do_layer_norm_before
123
@@ -127,16 +128,16 @@ class OPTDecoderLayer(nn.Module):
128 self.embed_dim,
129 config.ffn_dim,
130 bias=config.enable_bias,
130 - linear_method=linear_method,
131 + quant_config=quant_config,
132 )
132 - quant_config = getattr(linear_method, "quant_config", None)
133 + quant_config = getattr(quant_config, "quant_config", None)
134 self.activation_fn = get_act_fn(config.activation_function,
135 quant_config, config.ffn_dim)
136 self.fc2 = RowParallelLinear(
137 config.ffn_dim,
138 self.embed_dim,
139 bias=config.enable_bias,
139 - linear_method=linear_method,
140 + quant_config=quant_config,
141 )
142 self.final_layer_norm = nn.LayerNorm(
143 self.embed_dim,
@@ -181,7 +182,7 @@ class OPTDecoder(nn.Module):
182 def __init__(
183 self,
184 config: OPTConfig,
184 - linear_method: Optional[LinearMethodBase] = None,
185 + quant_config: Optional[QuantizationConfig] = None,
186 ):
187 super().__init__()
188 self.config = config
@@ -202,7 +203,7 @@ class OPTDecoder(nn.Module):
203 self.project_out = ReplicatedLinear(config.hidden_size,
204 config.word_embed_proj_dim,
205 bias=False,
205 - linear_method=linear_method)
206 + quant_config=quant_config)
207 else:
208 self.project_out = None
209
@@ -210,7 +211,7 @@ class OPTDecoder(nn.Module):
211 self.project_in = ReplicatedLinear(config.word_embed_proj_dim,
212 config.hidden_size,
213 bias=False,
213 - linear_method=linear_method)
214 + quant_config=quant_config)
215 else:
216 self.project_in = None
217
@@ -226,7 +227,7 @@ class OPTDecoder(nn.Module):
227 self.final_layer_norm = None
228
229 self.layers = nn.ModuleList([
229 - OPTDecoderLayer(config, linear_method)
230 + OPTDecoderLayer(config, quant_config)
231 for _ in range(config.num_hidden_layers)
232 ])
233
@@ -259,10 +260,10 @@ class OPTModel(nn.Module):
260 def __init__(
261 self,
262 config: OPTConfig,
262 - linear_method: Optional[LinearMethodBase] = None,
263 + quant_config: Optional[QuantizationConfig] = None,
264 ):
265 super().__init__()
265 - self.decoder = OPTDecoder(config, linear_method)
266 + self.decoder = OPTDecoder(config, quant_config)
267
268 def forward(
269 self,
@@ -279,12 +280,12 @@ class OPTForCausalLM(nn.Module):
280 def __init__(
281 self,
282 config,
282 - linear_method: Optional[LinearMethodBase] = None,
283 + quant_config: Optional[QuantizationConfig] = None,
284 ):
285 super().__init__()
286 self.config = config
286 - self.linear_method = linear_method
287 - self.model = OPTModel(config, linear_method)
287 + self.quant_config = quant_config
288 + self.model = OPTModel(config, quant_config)
289 self.lm_head_weight = self.model.decoder.embed_tokens.weight
290 self.logits_processor = LogitsProcessor(config.vocab_size)
291 self.sampler = Sampler()
vllm/model_executor/models/orion.py
+17 -16
@@ -13,11 +13,12 @@ from transformers import PretrainedConfig
13 from vllm.attention import Attention, AttentionMetadata
14 from vllm.distributed import get_tensor_model_parallel_world_size
15 from vllm.model_executor.layers.activation import SiluAndMul
16 -from vllm.model_executor.layers.linear import (LinearMethodBase,
17 - MergedColumnParallelLinear,
16 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
17 QKVParallelLinear,
18 RowParallelLinear)
19 from vllm.model_executor.layers.logits_processor import LogitsProcessor
20 +from vllm.model_executor.layers.quantization.base_config import (
21 + QuantizationConfig)
22 from vllm.model_executor.layers.rotary_embedding import get_rope
23 from vllm.model_executor.layers.sampler import Sampler
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -34,17 +35,17 @@ class OrionMLP(nn.Module):
35 hidden_size: int,
36 intermediate_size: int,
37 hidden_act: str,
37 - linear_method: Optional[LinearMethodBase] = None,
38 + quant_config: Optional[QuantizationConfig] = None,
39 ) -> None:
40 super().__init__()
41 self.gate_up_proj = MergedColumnParallelLinear(
42 hidden_size, [intermediate_size] * 2,
43 bias=False,
43 - linear_method=linear_method)
44 + quant_config=quant_config)
45 self.down_proj = RowParallelLinear(intermediate_size,
46 hidden_size,
47 bias=False,
47 - linear_method=linear_method)
48 + quant_config=quant_config)
49 if hidden_act != "silu":
50 raise ValueError(f"Unsupported activation: {hidden_act}. "
51 "Only silu is supported for now.")
@@ -67,7 +68,7 @@ class OrionAttention(nn.Module):
68 rope_theta: float = 10000,
69 rope_scaling: Optional[Dict[str, Any]] = None,
70 max_position_embeddings: int = 8192,
70 - linear_method: Optional[LinearMethodBase] = None,
71 + quant_config: Optional[QuantizationConfig] = None,
72 ) -> None:
73 super().__init__()
74 self.hidden_size = hidden_size
@@ -98,13 +99,13 @@ class OrionAttention(nn.Module):
99 self.total_num_heads,
100 self.total_num_kv_heads,
101 bias=False,
101 - linear_method=linear_method,
102 + quant_config=quant_config,
103 )
104 self.o_proj = RowParallelLinear(
105 self.total_num_heads * self.head_dim,
106 hidden_size,
107 bias=False,
107 - linear_method=linear_method,
108 + quant_config=quant_config,
109 )
110
111 self.rotary_emb = get_rope(
@@ -139,7 +140,7 @@ class OrionDecoderLayer(nn.Module):
140 def __init__(
141 self,
142 config: PretrainedConfig,
142 - linear_method: Optional[LinearMethodBase] = None,
143 + quant_config: Optional[QuantizationConfig] = None,
144 ) -> None:
145 super().__init__()
146 self.hidden_size = config.hidden_size
@@ -154,13 +155,13 @@ class OrionDecoderLayer(nn.Module):
155 rope_theta=rope_theta,
156 rope_scaling=rope_scaling,
157 max_position_embeddings=max_position_embeddings,
157 - linear_method=linear_method,
158 + quant_config=quant_config,
159 )
160 self.mlp = OrionMLP(
161 hidden_size=self.hidden_size,
162 intermediate_size=config.intermediate_size,
163 hidden_act=config.hidden_act,
163 - linear_method=linear_method,
164 + quant_config=quant_config,
165 )
166
167 self.input_layernorm = nn.LayerNorm(config.hidden_size,
@@ -201,7 +202,7 @@ class OrionModel(nn.Module):
202 def __init__(
203 self,
204 config: PretrainedConfig,
204 - linear_method: Optional[LinearMethodBase] = None,
205 + quant_config: Optional[QuantizationConfig] = None,
206 ) -> None:
207 super().__init__()
208 self.config = config
@@ -212,7 +213,7 @@ class OrionModel(nn.Module):
213 config.hidden_size,
214 )
215 self.layers = nn.ModuleList([
215 - OrionDecoderLayer(config, linear_method)
216 + OrionDecoderLayer(config, quant_config)
217 for _ in range(config.num_hidden_layers)
218 ])
219 self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -244,12 +245,12 @@ class OrionForCausalLM(nn.Module):
245 def __init__(
246 self,
247 config: PretrainedConfig,
247 - linear_method: Optional[LinearMethodBase] = None,
248 + quant_config: Optional[QuantizationConfig] = None,
249 ) -> None:
250 super().__init__()
251 self.config = config
251 - self.linear_method = linear_method
252 - self.model = OrionModel(config, linear_method)
252 + self.quant_config = quant_config
253 + self.model = OrionModel(config, quant_config)
254 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
255 self.logits_processor = LogitsProcessor(config.vocab_size)
256 self.sampler = Sampler()
vllm/model_executor/models/phi.py
+18 -17
@@ -45,10 +45,11 @@ from vllm.attention import Attention, AttentionMetadata
45 from vllm.distributed import get_tensor_model_parallel_world_size
46 from vllm.model_executor.layers.activation import get_act_fn
47 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
48 - LinearMethodBase,
48 QKVParallelLinear,
49 RowParallelLinear)
50 from vllm.model_executor.layers.logits_processor import LogitsProcessor
51 +from vllm.model_executor.layers.quantization.base_config import (
52 + QuantizationConfig)
53 from vllm.model_executor.layers.rotary_embedding import get_rope
54 from vllm.model_executor.layers.sampler import Sampler
55 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -62,7 +63,7 @@ class PhiAttention(nn.Module):
63
64 def __init__(self,
65 config: PretrainedConfig,
65 - linear_method: Optional[LinearMethodBase] = None):
66 + quant_config: Optional[QuantizationConfig] = None):
67 super().__init__()
68 self.total_num_heads = config.num_attention_heads
69 self.hidden_size = config.hidden_size
@@ -80,12 +81,12 @@ class PhiAttention(nn.Module):
81 self.head_size,
82 self.total_num_heads,
83 bias=True,
83 - linear_method=linear_method,
84 + quant_config=quant_config,
85 )
86 self.dense = RowParallelLinear(
87 self.hidden_size,
88 self.hidden_size,
88 - linear_method=linear_method,
89 + quant_config=quant_config,
90 )
91
92 scaling = self.head_size**-0.5
@@ -125,7 +126,7 @@ class PhiMLP(nn.Module):
126
127 def __init__(self,
128 config: PretrainedConfig,
128 - linear_method: Optional[LinearMethodBase] = None):
129 + quant_config: Optional[QuantizationConfig] = None):
130 super().__init__()
131
132 n_inner = getattr(config, "n_inner", None)
@@ -134,14 +135,14 @@ class PhiMLP(nn.Module):
135 self.fc1 = ColumnParallelLinear(
136 config.hidden_size,
137 n_inner,
137 - linear_method=linear_method,
138 + quant_config=quant_config,
139 )
140 self.fc2 = RowParallelLinear(
141 n_inner,
142 config.hidden_size,
142 - linear_method=linear_method,
143 + quant_config=quant_config,
144 )
144 - quant_config = getattr(linear_method, "quant_config", None)
145 + quant_config = getattr(quant_config, "quant_config", None)
146 self.act = get_act_fn(config.hidden_act, quant_config, n_inner)
147
148 def forward(self, hidden_states):
@@ -155,12 +156,12 @@ class PhiLayer(nn.Module):
156
157 def __init__(self,
158 config: PretrainedConfig,
158 - linear_method: Optional[LinearMethodBase] = None):
159 + quant_config: Optional[QuantizationConfig] = None):
160 super().__init__()
161 self.input_layernorm = nn.LayerNorm(config.hidden_size,
162 eps=config.layer_norm_eps)
162 - self.self_attn = PhiAttention(config, linear_method)
163 - self.mlp = PhiMLP(config, linear_method)
163 + self.self_attn = PhiAttention(config, quant_config)
164 + self.mlp = PhiMLP(config, quant_config)
165
166 def forward(
167 self,
@@ -186,14 +187,14 @@ class PhiModel(nn.Module):
187
188 def __init__(self,
189 config: PretrainedConfig,
189 - linear_method: Optional[LinearMethodBase] = None):
190 + quant_config: Optional[QuantizationConfig] = None):
191 super().__init__()
192 self.config = config
192 - self.linear_method = linear_method
193 + self.quant_config = quant_config
194 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
195 config.hidden_size)
196 self.layers = nn.ModuleList([
196 - PhiLayer(config, linear_method)
197 + PhiLayer(config, quant_config)
198 for _ in range(config.num_hidden_layers)
199 ])
200 self.final_layernorm = nn.LayerNorm(config.hidden_size,
@@ -225,12 +226,12 @@ class PhiForCausalLM(nn.Module):
226
227 def __init__(self,
228 config: PretrainedConfig,
228 - linear_method: Optional[LinearMethodBase] = None):
229 + quant_config: Optional[QuantizationConfig] = None):
230 super().__init__()
231 self.config = config
231 - self.linear_method = linear_method
232 + self.quant_config = quant_config
233
233 - self.model = PhiModel(config, linear_method)
234 + self.model = PhiModel(config, quant_config)
235
236 self.lm_head = ParallelLMHead(config.vocab_size,
237 config.hidden_size,
vllm/model_executor/models/qwen.py
+17 -16
@@ -14,11 +14,12 @@ from vllm.attention import Attention, AttentionMetadata
14 from vllm.distributed import get_tensor_model_parallel_world_size
15 from vllm.model_executor.layers.activation import SiluAndMul
16 from vllm.model_executor.layers.layernorm import RMSNorm
17 -from vllm.model_executor.layers.linear import (LinearMethodBase,
18 - MergedColumnParallelLinear,
17 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
18 QKVParallelLinear,
19 RowParallelLinear)
20 from vllm.model_executor.layers.logits_processor import LogitsProcessor
21 +from vllm.model_executor.layers.quantization.base_config import (
22 + QuantizationConfig)
23 from vllm.model_executor.layers.rotary_embedding import get_rope
24 from vllm.model_executor.layers.sampler import Sampler
25 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -35,17 +36,17 @@ class QWenMLP(nn.Module):
36 hidden_size: int,
37 intermediate_size: int,
38 hidden_act: str = "silu",
38 - linear_method: Optional[LinearMethodBase] = None,
39 + quant_config: Optional[QuantizationConfig] = None,
40 ):
41 super().__init__()
42 self.gate_up_proj = MergedColumnParallelLinear(
43 hidden_size, [intermediate_size] * 2,
44 bias=False,
44 - linear_method=linear_method)
45 + quant_config=quant_config)
46 self.c_proj = RowParallelLinear(intermediate_size,
47 hidden_size,
48 bias=False,
48 - linear_method=linear_method)
49 + quant_config=quant_config)
50 if hidden_act != "silu":
51 raise ValueError(f"Unsupported activation: {hidden_act}. "
52 "Only silu is supported for now.")
@@ -67,7 +68,7 @@ class QWenAttention(nn.Module):
68 max_position_embeddings: int,
69 rope_theta: float = 10000,
70 rope_scaling: Optional[Dict[str, Any]] = None,
70 - linear_method: Optional[LinearMethodBase] = None,
71 + quant_config: Optional[QuantizationConfig] = None,
72 ):
73 super().__init__()
74 self.hidden_size = hidden_size
@@ -83,13 +84,13 @@ class QWenAttention(nn.Module):
84 self.head_dim,
85 self.total_num_heads,
86 bias=True,
86 - linear_method=linear_method,
87 + quant_config=quant_config,
88 )
89 self.c_proj = RowParallelLinear(
90 self.total_num_heads * self.head_dim,
91 hidden_size,
92 bias=False,
92 - linear_method=linear_method,
93 + quant_config=quant_config,
94 )
95 self.scaling = self.head_dim**-0.5
96
@@ -122,7 +123,7 @@ class QWenBlock(nn.Module):
123 def __init__(
124 self,
125 config: PretrainedConfig,
125 - linear_method: Optional[LinearMethodBase] = None,
126 + quant_config: Optional[QuantizationConfig] = None,
127 ):
128 super().__init__()
129 self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -134,13 +135,13 @@ class QWenBlock(nn.Module):
135 config.max_position_embeddings,
136 rope_theta=rope_theta,
137 rope_scaling=rope_scaling,
137 - linear_method=linear_method)
138 + quant_config=quant_config)
139
140 self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
141
142 self.mlp = QWenMLP(config.hidden_size,
143 config.intermediate_size // 2,
143 - linear_method=linear_method)
144 + quant_config=quant_config)
145
146 def forward(
147 self,
@@ -174,7 +175,7 @@ class QWenModel(nn.Module):
175 def __init__(
176 self,
177 config: PretrainedConfig,
177 - linear_method: Optional[LinearMethodBase] = None,
178 + quant_config: Optional[QuantizationConfig] = None,
179 ):
180 super().__init__()
181 self.config = config
@@ -185,7 +186,7 @@ class QWenModel(nn.Module):
186 config.hidden_size,
187 )
188 self.h = nn.ModuleList([
188 - QWenBlock(config, linear_method)
189 + QWenBlock(config, quant_config)
190 for _ in range(config.num_hidden_layers)
191 ])
192 self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -217,12 +218,12 @@ class QWenLMHeadModel(nn.Module):
218 def __init__(
219 self,
220 config: PretrainedConfig,
220 - linear_method: Optional[LinearMethodBase] = None,
221 + quant_config: Optional[QuantizationConfig] = None,
222 ):
223 super().__init__()
224 self.config = config
224 - self.linear_method = linear_method
225 - self.transformer = QWenModel(config, linear_method)
225 + self.quant_config = quant_config
226 + self.transformer = QWenModel(config, quant_config)
227 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
228 self.logits_processor = LogitsProcessor(config.vocab_size)
229 self.sampler = Sampler()
vllm/model_executor/models/qwen2.py
+17 -16
@@ -33,11 +33,12 @@ from vllm.config import LoRAConfig
33 from vllm.distributed import get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
36 -from vllm.model_executor.layers.linear import (LinearMethodBase,
37 - MergedColumnParallelLinear,
36 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37 QKVParallelLinear,
38 RowParallelLinear)
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 +from vllm.model_executor.layers.quantization.base_config import (
41 + QuantizationConfig)
42 from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import Sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -54,17 +55,17 @@ class Qwen2MLP(nn.Module):
55 hidden_size: int,
56 intermediate_size: int,
57 hidden_act: str,
57 - linear_method: Optional[LinearMethodBase] = None,
58 + quant_config: Optional[QuantizationConfig] = None,
59 ) -> None:
60 super().__init__()
61 self.gate_up_proj = MergedColumnParallelLinear(
62 hidden_size, [intermediate_size] * 2,
63 bias=False,
63 - linear_method=linear_method)
64 + quant_config=quant_config)
65 self.down_proj = RowParallelLinear(intermediate_size,
66 hidden_size,
67 bias=False,
67 - linear_method=linear_method)
68 + quant_config=quant_config)
69 if hidden_act != "silu":
70 raise ValueError(f"Unsupported activation: {hidden_act}. "
71 "Only silu is supported for now.")
@@ -86,7 +87,7 @@ class Qwen2Attention(nn.Module):
87 max_position: int = 4096 * 32,
88 rope_theta: float = 10000,
89 use_sliding_window: bool = False,
89 - linear_method: Optional[LinearMethodBase] = None,
90 + quant_config: Optional[QuantizationConfig] = None,
91 sliding_window: Optional[int] = None) -> None:
92 super().__init__()
93 self.hidden_size = hidden_size
@@ -117,13 +118,13 @@ class Qwen2Attention(nn.Module):
118 self.total_num_heads,
119 self.total_num_kv_heads,
120 bias=True,
120 - linear_method=linear_method,
121 + quant_config=quant_config,
122 )
123 self.o_proj = RowParallelLinear(
124 self.total_num_heads * self.head_dim,
125 hidden_size,
126 bias=False,
126 - linear_method=linear_method,
127 + quant_config=quant_config,
128 )
129
130 self.rotary_emb = get_rope(
@@ -159,7 +160,7 @@ class Qwen2DecoderLayer(nn.Module):
160 self,
161 config: Qwen2Config,
162 layer_idx: int,
162 - linear_method: Optional[LinearMethodBase] = None,
163 + quant_config: Optional[QuantizationConfig] = None,
164 ) -> None:
165 super().__init__()
166 self.hidden_size = config.hidden_size
@@ -174,13 +175,13 @@ class Qwen2DecoderLayer(nn.Module):
175 num_kv_heads=config.num_key_value_heads,
176 rope_theta=rope_theta,
177 use_sliding_window=use_sliding_window,
177 - linear_method=linear_method,
178 + quant_config=quant_config,
179 sliding_window=config.sliding_window)
180 self.mlp = Qwen2MLP(
181 hidden_size=self.hidden_size,
182 intermediate_size=config.intermediate_size,
183 hidden_act=config.hidden_act,
183 - linear_method=linear_method,
184 + quant_config=quant_config,
185 )
186 self.input_layernorm = RMSNorm(config.hidden_size,
187 eps=config.rms_norm_eps)
@@ -221,7 +222,7 @@ class Qwen2Model(nn.Module):
222 def __init__(
223 self,
224 config: Qwen2Config,
224 - linear_method: Optional[LinearMethodBase] = None,
225 + quant_config: Optional[QuantizationConfig] = None,
226 ) -> None:
227 super().__init__()
228 self.config = config
@@ -233,7 +234,7 @@ class Qwen2Model(nn.Module):
234 config.hidden_size,
235 )
236 self.layers = nn.ModuleList([
236 - Qwen2DecoderLayer(config, layer_idx, linear_method)
237 + Qwen2DecoderLayer(config, layer_idx, quant_config)
238 for layer_idx in range(config.num_hidden_layers)
239 ])
240 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -286,14 +287,14 @@ class Qwen2ForCausalLM(nn.Module):
287 def __init__(
288 self,
289 config: Qwen2Config,
289 - linear_method: Optional[LinearMethodBase] = None,
290 + quant_config: Optional[QuantizationConfig] = None,
291 lora_config: Optional[LoRAConfig] = None,
292 ) -> None:
293 del lora_config
294 super().__init__()
295 self.config = config
295 - self.linear_method = linear_method
296 - self.model = Qwen2Model(config, linear_method)
296 + self.quant_config = quant_config
297 + self.model = Qwen2Model(config, quant_config)
298
299 if config.tie_word_embeddings:
300 self.lm_head_weight = self.model.embed_tokens.weight
vllm/model_executor/models/qwen2_moe.py
+22 -23
@@ -36,12 +36,13 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
36 from vllm.model_executor.layers.activation import SiluAndMul
37 from vllm.model_executor.layers.fused_moe import fused_moe
38 from vllm.model_executor.layers.layernorm import RMSNorm
39 -from vllm.model_executor.layers.linear import (LinearMethodBase,
40 - MergedColumnParallelLinear,
39 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
40 QKVParallelLinear,
41 ReplicatedLinear,
42 RowParallelLinear)
43 from vllm.model_executor.layers.logits_processor import LogitsProcessor
44 +from vllm.model_executor.layers.quantization.base_config import (
45 + QuantizationConfig)
46 from vllm.model_executor.layers.rotary_embedding import get_rope
47 from vllm.model_executor.layers.sampler import Sampler
48 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -58,18 +59,18 @@ class Qwen2MoeMLP(nn.Module):
59 hidden_size: int,
60 intermediate_size: int,
61 hidden_act: str,
61 - linear_method: Optional[LinearMethodBase] = None,
62 + quant_config: Optional[QuantizationConfig] = None,
63 reduce_results: bool = True,
64 ) -> None:
65 super().__init__()
66 self.gate_up_proj = MergedColumnParallelLinear(
67 hidden_size, [intermediate_size] * 2,
68 bias=False,
68 - linear_method=linear_method)
69 + quant_config=quant_config)
70 self.down_proj = RowParallelLinear(intermediate_size,
71 hidden_size,
72 bias=False,
72 - linear_method=linear_method,
73 + quant_config=quant_config,
74 reduce_results=reduce_results)
75 if hidden_act != "silu":
76 raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -88,7 +89,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
89 def __init__(
90 self,
91 config: PretrainedConfig,
91 - linear_method: Optional[LinearMethodBase] = None,
92 + quant_config: Optional[QuantizationConfig] = None,
93 ):
94 super().__init__()
95 self.config = config
@@ -105,7 +106,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
106 Qwen2MoeMLP(hidden_size=config.hidden_size,
107 intermediate_size=config.moe_intermediate_size,
108 hidden_act=config.hidden_act,
108 - linear_method=linear_method,
109 + quant_config=quant_config,
110 reduce_results=False)
111 for idx in range(self.n_routed_experts)
112 ])
@@ -114,13 +115,13 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
115 self.gate = ReplicatedLinear(config.hidden_size,
116 self.n_routed_experts,
117 bias=False,
117 - linear_method=None)
118 + quant_config=None)
119 if config.shared_expert_intermediate_size > 0:
120 self.shared_expert = Qwen2MoeMLP(
121 hidden_size=config.hidden_size,
122 intermediate_size=config.shared_expert_intermediate_size,
123 hidden_act=config.hidden_act,
123 - linear_method=linear_method,
124 + quant_config=quant_config,
125 reduce_results=False,
126 )
127 else:
@@ -186,7 +187,7 @@ class Qwen2MoeAttention(nn.Module):
187 rope_theta: float = 10000,
188 rope_scaling: Optional[Dict[str, Any]] = None,
189 max_position_embeddings: int = 8192,
189 - linear_method: Optional[LinearMethodBase] = None,
190 + quant_config: Optional[QuantizationConfig] = None,
191 ) -> None:
192 super().__init__()
193 self.hidden_size = hidden_size
@@ -217,14 +218,14 @@ class Qwen2MoeAttention(nn.Module):
218 self.total_num_heads,
219 self.total_num_kv_heads,
220 bias=True,
220 - linear_method=linear_method,
221 + quant_config=quant_config,
222 )
223
224 self.o_proj = RowParallelLinear(
225 self.total_num_heads * self.head_dim,
226 hidden_size,
227 bias=False,
227 - linear_method=linear_method,
228 + quant_config=quant_config,
229 )
230
231 self.rotary_emb = get_rope(
@@ -260,7 +261,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
261 self,
262 config: PretrainedConfig,
263 layer_idx: int,
263 - linear_method: Optional[LinearMethodBase] = None,
264 + quant_config: Optional[QuantizationConfig] = None,
265 ) -> None:
266 super().__init__()
267 self.hidden_size = config.hidden_size
@@ -275,18 +276,18 @@ class Qwen2MoeDecoderLayer(nn.Module):
276 rope_theta=rope_theta,
277 rope_scaling=rope_scaling,
278 max_position_embeddings=max_position_embeddings,
278 - linear_method=linear_method,
279 + quant_config=quant_config,
280 )
281 if (config.num_experts is not None
282 and (layer_idx + 1) % config.decoder_sparse_step == 0):
283 self.mlp = Qwen2MoeSparseMoeBlock(config=config,
283 - linear_method=linear_method)
284 + quant_config=quant_config)
285 else:
286 self.mlp = Qwen2MoeMLP(
287 hidden_size=config.hidden_size,
288 intermediate_size=config.intermediate_size,
289 hidden_act=config.hidden_act,
289 - linear_method=linear_method,
290 + quant_config=quant_config,
291 )
292 self.input_layernorm = RMSNorm(config.hidden_size,
293 eps=config.rms_norm_eps)
@@ -327,7 +328,7 @@ class Qwen2MoeModel(nn.Module):
328 def __init__(
329 self,
330 config: PretrainedConfig,
330 - linear_method: Optional[LinearMethodBase] = None,
331 + quant_config: Optional[QuantizationConfig] = None,
332 ) -> None:
333 super().__init__()
334 self.padding_idx = config.pad_token_id
@@ -338,9 +339,7 @@ class Qwen2MoeModel(nn.Module):
339 config.hidden_size,
340 )
341 self.layers = nn.ModuleList([
341 - Qwen2MoeDecoderLayer(config,
342 - layer_idx,
343 - linear_method=linear_method)
342 + Qwen2MoeDecoderLayer(config, layer_idx, quant_config=quant_config)
343 for layer_idx in range(config.num_hidden_layers)
344 ])
345 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -370,12 +369,12 @@ class Qwen2MoeForCausalLM(nn.Module):
369 def __init__(
370 self,
371 config: PretrainedConfig,
373 - linear_method: Optional[LinearMethodBase] = None,
372 + quant_config: Optional[QuantizationConfig] = None,
373 ) -> None:
374 super().__init__()
375 self.config = config
377 - self.linear_method = linear_method
378 - self.model = Qwen2MoeModel(config, linear_method)
376 + self.quant_config = quant_config
377 + self.model = Qwen2MoeModel(config, quant_config)
378 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
379 self.logits_processor = LogitsProcessor(config.vocab_size)
380 self.sampler = Sampler()
vllm/model_executor/models/stablelm.py
+15 -14
@@ -28,11 +28,12 @@ from transformers import PretrainedConfig
28 from vllm.attention import Attention, AttentionMetadata
29 from vllm.distributed import get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import SiluAndMul
31 -from vllm.model_executor.layers.linear import (LinearMethodBase,
32 - MergedColumnParallelLinear,
31 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
32 QKVParallelLinear,
33 RowParallelLinear)
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 +from vllm.model_executor.layers.quantization.base_config import (
36 + QuantizationConfig)
37 from vllm.model_executor.layers.rotary_embedding import get_rope
38 from vllm.model_executor.layers.sampler import Sampler
39 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -46,7 +47,7 @@ class StablelmMLP(nn.Module):
47
48 def __init__(self,
49 config: PretrainedConfig,
49 - linear_method: Optional[LinearMethodBase] = None) -> None:
50 + quant_config: Optional[QuantizationConfig] = None) -> None:
51 super().__init__()
52 self.config = config
53 self.hidden_size = config.hidden_size
@@ -54,7 +55,7 @@ class StablelmMLP(nn.Module):
55 self.gate_up_proj = MergedColumnParallelLinear(
56 config.hidden_size, [config.intermediate_size] * 2,
57 bias=False,
57 - linear_method=linear_method)
58 + quant_config=quant_config)
59 self.down_proj = RowParallelLinear(config.intermediate_size,
60 config.hidden_size,
61 bias=False)
@@ -71,7 +72,7 @@ class StablelmAttention(nn.Module):
72
73 def __init__(self,
74 config: PretrainedConfig,
74 - linear_method: Optional[LinearMethodBase] = None) -> None:
75 + quant_config: Optional[QuantizationConfig] = None) -> None:
76 super().__init__()
77 self.config = config
78 self.hidden_size = config.hidden_size
@@ -109,11 +110,11 @@ class StablelmAttention(nn.Module):
110 self.total_num_heads,
111 self.total_num_key_value_heads,
112 self.qkv_bias,
112 - linear_method=linear_method)
113 + quant_config=quant_config)
114 self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
115 self.hidden_size,
116 bias=False,
116 - linear_method=linear_method)
117 + quant_config=quant_config)
118 self.rotary_emb = get_rope(
119 self.head_dim,
120 rotary_dim=self.rotary_ndims,
@@ -145,11 +146,11 @@ class StablelmDecoderLayer(nn.Module):
146 def __init__(
147 self,
148 config: PretrainedConfig,
148 - linear_method: Optional[LinearMethodBase] = None,
149 + quant_config: Optional[QuantizationConfig] = None,
150 ) -> None:
151 super().__init__()
152 self.self_attn = StablelmAttention(config)
152 - self.mlp = StablelmMLP(config, linear_method)
153 + self.mlp = StablelmMLP(config, quant_config)
154 norm_eps = getattr(config, "norm_eps",
155 getattr(config, "layer_norm_eps", 1e-05))
156 self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
@@ -187,14 +188,14 @@ class StableLMEpochModel(nn.Module):
188
189 def __init__(self,
190 config: PretrainedConfig,
190 - linear_method: Optional[LinearMethodBase] = None) -> None:
191 + quant_config: Optional[QuantizationConfig] = None) -> None:
192 super().__init__()
193 self.embed_tokens = VocabParallelEmbedding(
194 config.vocab_size,
195 config.hidden_size,
196 )
197 self.layers = nn.ModuleList([
197 - StablelmDecoderLayer(config, linear_method)
198 + StablelmDecoderLayer(config, quant_config)
199 for _ in range(config.num_hidden_layers)
200 ])
201 norm_eps = getattr(config, "norm_eps",
@@ -226,12 +227,12 @@ class StablelmForCausalLM(nn.Module):
227 def __init__(
228 self,
229 config: PretrainedConfig,
229 - linear_method: Optional[LinearMethodBase] = None,
230 + quant_config: Optional[QuantizationConfig] = None,
231 ) -> None:
232 super().__init__()
233 self.config = config
233 - self.linear_method = linear_method
234 - self.model = StableLMEpochModel(config, linear_method)
234 + self.quant_config = quant_config
235 + self.model = StableLMEpochModel(config, quant_config)
236 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
237 self.logits_processor = LogitsProcessor(config.vocab_size)
238 self.sampler = Sampler()
vllm/model_executor/models/starcoder2.py
+16 -16
@@ -28,10 +28,11 @@ from vllm.attention import Attention, AttentionMetadata
28 from vllm.distributed import get_tensor_model_parallel_world_size
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 - LinearMethodBase,
31 QKVParallelLinear,
32 RowParallelLinear)
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 +from vllm.model_executor.layers.quantization.base_config import (
35 + QuantizationConfig)
36 from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -45,7 +46,7 @@ class Starcoder2Attention(nn.Module):
46
47 def __init__(self,
48 config: Starcoder2Config,
48 - linear_method: Optional[LinearMethodBase] = None):
49 + quant_config: Optional[QuantizationConfig] = None):
50 super().__init__()
51 self.config = config
52
@@ -79,13 +80,13 @@ class Starcoder2Attention(nn.Module):
80 self.total_num_heads,
81 self.total_num_kv_heads,
82 bias=self.use_bias,
82 - linear_method=linear_method,
83 + quant_config=quant_config,
84 )
85 self.o_proj = RowParallelLinear(
86 self.total_num_heads * self.head_dim,
87 self.hidden_size,
88 bias=self.use_bias,
88 - linear_method=linear_method,
89 + quant_config=quant_config,
90 )
91 self.rotary_emb = get_rope(
92 self.head_dim,
@@ -121,21 +122,21 @@ class Starcoder2MLP(nn.Module):
122
123 def __init__(self,
124 config: Starcoder2Config,
124 - linear_method: Optional[LinearMethodBase] = None):
125 + quant_config: Optional[QuantizationConfig] = None):
126 super().__init__()
127 self.c_fc = ColumnParallelLinear(
128 config.hidden_size,
129 config.intermediate_size,
130 bias=config.use_bias,
130 - linear_method=linear_method,
131 + quant_config=quant_config,
132 )
133 self.c_proj = RowParallelLinear(
134 config.intermediate_size,
135 config.hidden_size,
136 bias=config.use_bias,
136 - linear_method=linear_method,
137 + quant_config=quant_config,
138 )
138 - quant_config = getattr(linear_method, "quant_config", None)
139 + quant_config = getattr(quant_config, "quant_config", None)
140 self.act = get_act_fn(config.hidden_act, quant_config,
141 config.intermediate_size)
142
@@ -150,12 +151,11 @@ class Starcoder2DecoderLayer(nn.Module):
151
152 def __init__(self,
153 config: Starcoder2Config,
153 - linear_method: Optional[LinearMethodBase] = None):
154 + quant_config: Optional[QuantizationConfig] = None):
155 super().__init__()
156 self.hidden_size = config.hidden_size
156 - self.self_attn = Starcoder2Attention(config,
157 - linear_method=linear_method)
158 - self.mlp = Starcoder2MLP(config, linear_method=linear_method)
157 + self.self_attn = Starcoder2Attention(config, quant_config=quant_config)
158 + self.mlp = Starcoder2MLP(config, quant_config=quant_config)
159 self.input_layernorm = nn.LayerNorm(config.hidden_size,
160 eps=config.norm_epsilon)
161 self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
@@ -192,7 +192,7 @@ class Starcoder2Model(nn.Module):
192
193 def __init__(self,
194 config: Starcoder2Config,
195 - linear_method: Optional[LinearMethodBase] = None):
195 + quant_config: Optional[QuantizationConfig] = None):
196 super().__init__()
197 self.config = config
198 self.padding_idx = config.pad_token_id
@@ -202,7 +202,7 @@ class Starcoder2Model(nn.Module):
202 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
203 config.hidden_size)
204 self.layers = nn.ModuleList([
205 - Starcoder2DecoderLayer(config, linear_method=linear_method)
205 + Starcoder2DecoderLayer(config, quant_config=quant_config)
206 for _ in range(config.num_hidden_layers)
207 ])
208 self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon)
@@ -227,10 +227,10 @@ class Starcoder2ForCausalLM(nn.Module):
227
228 def __init__(self,
229 config: Starcoder2Config,
230 - linear_method: Optional[LinearMethodBase] = None):
230 + quant_config: Optional[QuantizationConfig] = None):
231 super().__init__()
232 self.config = config
233 - self.model = Starcoder2Model(config, linear_method=linear_method)
233 + self.model = Starcoder2Model(config, quant_config=quant_config)
234 self.vocab_size = config.vocab_size
235 self.unpadded_vocab_size = config.vocab_size
236 if config.tie_word_embeddings:
vllm/model_executor/models/xverse.py
+17 -16
@@ -31,11 +31,12 @@ from vllm.config import LoRAConfig
31 from vllm.distributed import get_tensor_model_parallel_world_size
32 from vllm.model_executor.layers.activation import SiluAndMul
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 -from vllm.model_executor.layers.linear import (LinearMethodBase,
35 - MergedColumnParallelLinear,
34 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
35 QKVParallelLinear,
36 RowParallelLinear)
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 +from vllm.model_executor.layers.quantization.base_config import (
39 + QuantizationConfig)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -52,17 +53,17 @@ class XverseMLP(nn.Module):
53 hidden_size: int,
54 intermediate_size: int,
55 hidden_act: str,
55 - linear_method: Optional[LinearMethodBase] = None,
56 + quant_config: Optional[QuantizationConfig] = None,
57 ) -> None:
58 super().__init__()
59 self.gate_up_proj = MergedColumnParallelLinear(
60 hidden_size, [intermediate_size] * 2,
61 bias=False,
61 - linear_method=linear_method)
62 + quant_config=quant_config)
63 self.down_proj = RowParallelLinear(intermediate_size,
64 hidden_size,
65 bias=False,
65 - linear_method=linear_method)
66 + quant_config=quant_config)
67 if hidden_act != "silu":
68 raise ValueError(f"Unsupported activation: {hidden_act}. "
69 "Only silu is supported for now.")
@@ -85,7 +86,7 @@ class XverseAttention(nn.Module):
86 rope_theta: float = 10000,
87 rope_scaling: Optional[Dict[str, Any]] = None,
88 max_position_embeddings: int = 8192,
88 - linear_method: Optional[LinearMethodBase] = None,
89 + quant_config: Optional[QuantizationConfig] = None,
90 bias: bool = False,
91 sliding_window: Optional[int] = None,
92 ) -> None:
@@ -112,13 +113,13 @@ class XverseAttention(nn.Module):
113 self.total_num_heads,
114 self.total_num_kv_heads,
115 bias=bias,
115 - linear_method=linear_method,
116 + quant_config=quant_config,
117 )
118 self.o_proj = RowParallelLinear(
119 self.total_num_heads * self.head_dim,
120 hidden_size,
121 bias=bias,
121 - linear_method=linear_method,
122 + quant_config=quant_config,
123 )
124
125 self.rotary_emb = get_rope(
@@ -154,7 +155,7 @@ class XverseDecoderLayer(nn.Module):
155 def __init__(
156 self,
157 config: PretrainedConfig,
157 - linear_method: Optional[LinearMethodBase] = None,
158 + quant_config: Optional[QuantizationConfig] = None,
159 ) -> None:
160 super().__init__()
161 self.hidden_size = config.hidden_size
@@ -171,7 +172,7 @@ class XverseDecoderLayer(nn.Module):
172 rope_theta=rope_theta,
173 rope_scaling=rope_scaling,
174 max_position_embeddings=max_position_embeddings,
174 - linear_method=linear_method,
175 + quant_config=quant_config,
176 bias=getattr(config, "bias", False),
177 sliding_window=sliding_window,
178 )
@@ -179,7 +180,7 @@ class XverseDecoderLayer(nn.Module):
180 hidden_size=self.hidden_size,
181 intermediate_size=config.intermediate_size,
182 hidden_act=config.hidden_act,
182 - linear_method=linear_method,
183 + quant_config=quant_config,
184 )
185 self.input_layernorm = RMSNorm(config.hidden_size,
186 eps=config.rms_norm_eps)
@@ -220,7 +221,7 @@ class XverseModel(nn.Module):
221 def __init__(
222 self,
223 config: PretrainedConfig,
223 - linear_method: Optional[LinearMethodBase] = None,
224 + quant_config: Optional[QuantizationConfig] = None,
225 lora_config: Optional[LoRAConfig] = None,
226 ) -> None:
227 super().__init__()
@@ -236,7 +237,7 @@ class XverseModel(nn.Module):
237 org_num_embeddings=config.vocab_size,
238 )
239 self.layers = nn.ModuleList([
239 - XverseDecoderLayer(config, linear_method)
240 + XverseDecoderLayer(config, quant_config)
241 for _ in range(config.num_hidden_layers)
242 ])
243 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -294,13 +295,13 @@ class XverseForCausalLM(nn.Module):
295 def __init__(
296 self,
297 config: PretrainedConfig,
297 - linear_method: Optional[LinearMethodBase] = None,
298 + quant_config: Optional[QuantizationConfig] = None,
299 lora_config=None,
300 ) -> None:
301 super().__init__()
302 self.config = config
302 - self.linear_method = linear_method
303 - self.model = XverseModel(config, linear_method)
303 + self.quant_config = quant_config
304 + self.model = XverseModel(config, quant_config)
305 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
306 self.logits_processor = LogitsProcessor(config.vocab_size)
307 self.sampler = Sampler()