A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc][Refactor] Generalize linear_method to be quant_method (#4373)
Cody Yu committed
Apr 26, 2024 at 13:41 UTC
a62aaf1df558d69658a42c1ab749368ab0325f35
45 files changed
+759
-713
tests/quantization/test_fp8.py
+1
-1
@@ -20,5 +20,5 @@ def test_load_fp16_model(vllm_runner) -> None:
20
21
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
22
fc1 = model.model.decoder.layers[0].fc1
23
- assert isinstance(fc1.linear_method, Fp8LinearMethod)
23
+ assert isinstance(fc1.quant_method, Fp8LinearMethod)
24
assert fc1.weight.dtype == torch.float8_e4m3fn
tests/tensorizer_loader/test_tensorizer.py
+2
-2
@@ -50,10 +50,10 @@ def test_load_with_tensorizer(mock_agent, tensorizer_config):
50
mock_agent_instance.deserialize.return_value = MagicMock()
51
52
result = load_with_tensorizer(tensorizer_config,
53
- linear_method=mock_linear_method)
53
+ quant_method=mock_linear_method)
54
55
mock_agent.assert_called_once_with(tensorizer_config,
56
- linear_method=mock_linear_method)
56
+ quant_method=mock_linear_method)
57
mock_agent_instance.deserialize.assert_called_once()
58
assert result == mock_agent_instance.deserialize.return_value
59
vllm/lora/layers.py
+13
-17
@@ -389,10 +389,9 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
389
self.indices = base_indices
390
self.indices_len = indices_len
391
392
- def apply_weights(self, x: torch.Tensor,
393
- bias: Optional[torch.Tensor]) -> torch.Tensor:
394
- output = self.base_layer.linear_method.apply_weights(
395
- self.base_layer, x, bias)
392
+ def apply(self, x: torch.Tensor,
393
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
394
+ output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
395
_apply_lora(
396
x,
397
self.lora_a_stacked,
@@ -416,7 +415,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
415
if not self.base_layer.skip_bias_add else None)
416
417
# Matrix multiply.
419
- output_parallel = self.apply_weights(input_, bias)
418
+ output_parallel = self.apply(input_, bias)
419
if self.base_layer.gather_output:
420
# All-gather across the partitions.
421
output = tensor_model_parallel_all_gather(output_parallel)
@@ -523,10 +522,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
522
index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
523
lora_b[1].T, non_blocking=True)
524
526
- def apply_weights(self, x: torch.Tensor,
527
- bias: Optional[torch.Tensor]) -> torch.Tensor:
528
- output = self.base_layer.linear_method.apply_weights(
529
- self.base_layer, x, bias)
525
+ def apply(self, x: torch.Tensor,
526
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
527
+ output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
528
_apply_lora_packed_nslice(
529
x,
530
self.lora_a_stacked,
@@ -765,10 +763,9 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
763
index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
764
lora_a[2].T, non_blocking=True)
765
768
- def apply_weights(self, x: torch.Tensor,
769
- bias: Optional[torch.Tensor]) -> torch.Tensor:
770
- output = self.base_layer.linear_method.apply_weights(
771
- self.base_layer, x, bias)
766
+ def apply(self, x: torch.Tensor,
767
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
768
+ output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
769
_apply_lora_packed_nslice(
770
x,
771
self.lora_a_stacked,
@@ -862,9 +859,8 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
859
self.indices = base_indices
860
self.indices_len = indices_len
861
865
- def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
866
- output = self.base_layer.linear_method.apply_weights(
867
- self.base_layer, x)
862
+ def apply(self, x: torch.Tensor) -> torch.Tensor:
863
+ output = self.base_layer.quant_method.apply(self.base_layer, x)
864
_apply_lora(
865
x,
866
self.lora_a_stacked,
@@ -897,7 +893,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
893
input_parallel = splitted_input[tp_rank].contiguous()
894
895
# Matrix multiply.
900
- output_parallel = self.apply_weights(input_parallel)
896
+ output_parallel = self.apply(input_parallel)
897
if self.base_layer.reduce_results and self.base_layer.tp_size > 1:
898
output_ = tensor_model_parallel_all_reduce(output_parallel)
899
else:
vllm/model_executor/layers/linear.py
+88
-81
@@ -1,9 +1,8 @@
1
-from abc import ABC, abstractmethod
1
+from abc import abstractmethod
2
from typing import List, Optional
3
4
import torch
5
import torch.nn.functional as F
6
-from torch import nn
6
from torch.nn.parameter import Parameter
7
8
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
@@ -12,6 +11,8 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
11
tensor_model_parallel_all_gather,
12
tensor_model_parallel_all_reduce)
13
from vllm.logger import init_logger
14
+from vllm.model_executor.layers.quantization.base_config import (
15
+ QuantizationConfig, QuantizeMethodBase)
16
from vllm.model_executor.utils import set_weight_attrs
17
18
logger = init_logger(__name__)
@@ -25,7 +26,7 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
26
return shard_size * marlin_tile_size, shard_offset * marlin_tile_size
27
28
28
-class LinearMethodBase(ABC):
29
+class LinearMethodBase(QuantizeMethodBase):
30
"""Base class for different (maybe quantized) linear methods."""
31
32
@abstractmethod
@@ -50,22 +51,15 @@ class LinearMethodBase(ABC):
51
raise NotImplementedError
52
53
@abstractmethod
53
- def apply_weights(self,
54
- layer: torch.nn.Module,
55
- x: torch.Tensor,
56
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
54
+ def apply(self,
55
+ layer: torch.nn.Module,
56
+ x: torch.Tensor,
57
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
58
"""Apply the weights in layer to the input tensor.
59
60
Expects create_weights to have been called before on the layer."""
61
raise NotImplementedError
62
62
- def process_weights_after_loading(self, layer: nn.Module) -> None:
63
- """Process the weight after loading.
64
-
65
- This can be used for example, to transpose weights for computation.
66
- """
67
- return
68
-
63
64
class UnquantizedLinearMethod(LinearMethodBase):
65
"""Linear method without quantization.
@@ -92,10 +86,10 @@ class UnquantizedLinearMethod(LinearMethodBase):
86
layer.register_parameter("weight", weight)
87
set_weight_attrs(weight, extra_weight_attrs)
88
95
- def apply_weights(self,
96
- layer: torch.nn.Module,
97
- x: torch.Tensor,
98
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
89
+ def apply(self,
90
+ layer: torch.nn.Module,
91
+ x: torch.Tensor,
92
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
93
weight = layer.weight
94
if self.separate_bias_add:
95
if bias is not None:
@@ -104,8 +98,8 @@ class UnquantizedLinearMethod(LinearMethodBase):
98
return F.linear(x, weight, bias)
99
100
107
-class ReplicatedLinear(torch.nn.Module):
108
- """Replicated linear layer.
101
+class LinearBase(torch.nn.Module):
102
+ """Base linear layer.
103
104
Args:
105
input_size: input dimension of the linear layer.
@@ -113,17 +107,16 @@ class ReplicatedLinear(torch.nn.Module):
107
bias: If true, add bias.
108
skip_bias_add: If true, skip adding bias but instead return it.
109
params_dtype: Data type for the parameters.
116
- linear_method: (Maybe quantized) linear method.
110
+ quant_config: Quantization configure.
111
"""
112
113
def __init__(
114
self,
115
input_size: int,
116
output_size: int,
123
- bias: bool = True,
117
skip_bias_add: bool = False,
118
params_dtype: Optional[torch.dtype] = None,
126
- linear_method: Optional[LinearMethodBase] = None,
119
+ quant_config: Optional[QuantizationConfig] = None,
120
):
121
super().__init__()
122
@@ -134,12 +127,43 @@ class ReplicatedLinear(torch.nn.Module):
127
if params_dtype is None:
128
params_dtype = torch.get_default_dtype()
129
self.params_dtype = params_dtype
137
- if linear_method is None:
138
- linear_method = UnquantizedLinearMethod()
139
- self.linear_method = linear_method
140
- self.linear_method.create_weights(self, self.input_size,
141
- [self.output_size], self.input_size,
142
- self.output_size, self.params_dtype)
130
+ if quant_config is None:
131
+ self.quant_method = UnquantizedLinearMethod()
132
+ else:
133
+ self.quant_method = quant_config.get_quant_method(self)
134
+
135
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
136
+ raise NotImplementedError
137
+
138
+
139
+class ReplicatedLinear(LinearBase):
140
+ """Replicated linear layer.
141
+
142
+ Args:
143
+ input_size: input dimension of the linear layer.
144
+ output_size: output dimension of the linear layer.
145
+ bias: If true, add bias.
146
+ skip_bias_add: If true, skip adding bias but instead return it.
147
+ params_dtype: Data type for the parameters.
148
+ quant_config: Quantization configure.
149
+ """
150
+
151
+ def __init__(
152
+ self,
153
+ input_size: int,
154
+ output_size: int,
155
+ bias: bool = True,
156
+ skip_bias_add: bool = False,
157
+ params_dtype: Optional[torch.dtype] = None,
158
+ quant_config: Optional[QuantizationConfig] = None,
159
+ ):
160
+ super().__init__(input_size, output_size, skip_bias_add, params_dtype,
161
+ quant_config)
162
+
163
+ self.quant_method.create_weights(self, self.input_size,
164
+ [self.output_size], self.input_size,
165
+ self.output_size, self.params_dtype)
166
+
167
if bias:
168
self.bias = Parameter(
169
torch.empty(self.output_size, dtype=self.params_dtype))
@@ -149,12 +173,12 @@ class ReplicatedLinear(torch.nn.Module):
173
174
def forward(self, x: torch.Tensor) -> torch.Tensor:
175
bias = self.bias if not self.skip_bias_add else None
152
- output = self.linear_method.apply_weights(self, x, bias)
176
+ output = self.quant_method.apply(self, x, bias)
177
output_bias = self.bias if self.skip_bias_add else None
178
return output, output_bias
179
180
157
-class ColumnParallelLinear(torch.nn.Module):
181
+class ColumnParallelLinear(LinearBase):
182
"""Linear layer with column parallelism.
183
184
The linear layer is defined as Y = XA + b. A is parallelized along
@@ -171,7 +195,7 @@ class ColumnParallelLinear(torch.nn.Module):
195
bias can be fused with other element-wise operations. we
196
skip adding bias but instead return it.
197
params_dtype: Data type for the parameters.
174
- linear_method: (Maybe quantized) linear method.
198
+ quant_config: Quantization configure.
199
output_sizes: list of output sizes packed into one output, like for QKV
200
the list would be size 3.
201
"""
@@ -184,34 +208,26 @@ class ColumnParallelLinear(torch.nn.Module):
208
gather_output: bool = False,
209
skip_bias_add: bool = False,
210
params_dtype: Optional[torch.dtype] = None,
187
- linear_method: Optional[LinearMethodBase] = None,
211
+ quant_config: Optional[QuantizationConfig] = None,
212
output_sizes: Optional[List[int]] = None,
213
):
190
- super().__init__()
214
+ super().__init__(input_size, output_size, skip_bias_add, params_dtype,
215
+ quant_config)
216
192
- # Keep input parameters
193
- self.input_size = input_size
194
- self.output_size = output_size
217
self.gather_output = gather_output
218
+
219
# Divide the weight matrix along the last dimension.
220
tp_size = get_tensor_model_parallel_world_size()
221
self.output_size_per_partition = divide(output_size, tp_size)
199
- self.skip_bias_add = skip_bias_add
200
- if params_dtype is None:
201
- params_dtype = torch.get_default_dtype()
202
- self.params_dtype = params_dtype
203
- if linear_method is None:
204
- linear_method = UnquantizedLinearMethod()
222
if output_sizes is None:
223
output_sizes = [output_size]
207
- self.linear_method = linear_method
208
- self.linear_method.create_weights(self,
209
- self.input_size,
210
- [x // tp_size for x in output_sizes],
211
- self.input_size,
212
- self.output_size,
213
- self.params_dtype,
214
- weight_loader=self.weight_loader)
224
+ self.quant_method.create_weights(self,
225
+ self.input_size,
226
+ [x // tp_size for x in output_sizes],
227
+ self.input_size,
228
+ self.output_size,
229
+ self.params_dtype,
230
+ weight_loader=self.weight_loader)
231
if bias:
232
self.bias = Parameter(
233
torch.empty(self.output_size_per_partition,
@@ -239,7 +255,7 @@ class ColumnParallelLinear(torch.nn.Module):
255
bias = self.bias if not self.skip_bias_add else None
256
257
# Matrix multiply.
242
- output_parallel = self.linear_method.apply_weights(self, input_, bias)
258
+ output_parallel = self.quant_method.apply(self, input_, bias)
259
if self.gather_output:
260
# All-gather across the partitions.
261
output = tensor_model_parallel_all_gather(output_parallel)
@@ -267,7 +283,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
283
bias can be fused with other element-wise operations. we
284
skip adding bias but instead return it.
285
params_dtype: Data type for the parameters.
270
- linear_method: (Maybe quantized) linear method.
286
+ quant_config: Quantization configure.
287
"""
288
289
def __init__(
@@ -278,13 +294,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
294
gather_output: bool = False,
295
skip_bias_add: bool = False,
296
params_dtype: Optional[torch.dtype] = None,
281
- linear_method: Optional[LinearMethodBase] = None,
297
+ quant_config: Optional[QuantizationConfig] = None,
298
):
299
self.output_sizes = output_sizes
300
tp_size = get_tensor_model_parallel_world_size()
301
assert all(output_size % tp_size == 0 for output_size in output_sizes)
302
super().__init__(input_size, sum(output_sizes), bias, gather_output,
287
- skip_bias_add, params_dtype, linear_method,
303
+ skip_bias_add, params_dtype, quant_config,
304
self.output_sizes)
305
306
def weight_loader(self,
@@ -384,7 +400,7 @@ class QKVParallelLinear(ColumnParallelLinear):
400
bias can be fused with other element-wise operations. we
401
skip adding bias but instead return it.
402
params_dtype: Data type for the parameters.
387
- linear_method: (Maybe quantized) linear method.
403
+ quant_config: Quantization configure.
404
"""
405
406
def __init__(
@@ -396,7 +412,7 @@ class QKVParallelLinear(ColumnParallelLinear):
412
bias: bool = True,
413
skip_bias_add: bool = False,
414
params_dtype: Optional[torch.dtype] = None,
399
- linear_method: Optional[LinearMethodBase] = None,
415
+ quant_config: Optional[QuantizationConfig] = None,
416
):
417
self.hidden_size = hidden_size
418
self.head_size = head_size
@@ -424,7 +440,7 @@ class QKVParallelLinear(ColumnParallelLinear):
440
]
441
442
super().__init__(input_size, output_size, bias, False, skip_bias_add,
427
- params_dtype, linear_method, output_sizes)
443
+ params_dtype, quant_config, output_sizes)
444
445
def weight_loader(self,
446
param: Parameter,
@@ -517,7 +533,7 @@ class QKVParallelLinear(ColumnParallelLinear):
533
param_data.copy_(loaded_weight)
534
535
520
-class RowParallelLinear(torch.nn.Module):
536
+class RowParallelLinear(LinearBase):
537
"""Linear layer with row parallelism.
538
539
The linear layer is defined as Y = XA + b. A is parallelized along
@@ -540,7 +556,7 @@ class RowParallelLinear(torch.nn.Module):
556
bias can be fused with other element-wise operations.
557
We skip adding bias but instead return it.
558
params_dtype: Data type for the parameters.
543
- linear_method: (Maybe quantized) linear method.
559
+ quant_config: Quantization configure.
560
"""
561
562
def __init__(
@@ -552,32 +568,24 @@ class RowParallelLinear(torch.nn.Module):
568
skip_bias_add: bool = False,
569
params_dtype: Optional[torch.dtype] = None,
570
reduce_results: bool = True,
555
- linear_method: Optional[LinearMethodBase] = None,
571
+ quant_config: Optional[QuantizationConfig] = None,
572
):
557
- super().__init__()
558
- # Keep input parameters
559
- self.input_size = input_size
560
- self.output_size = output_size
573
+ super().__init__(input_size, output_size, skip_bias_add, params_dtype,
574
+ quant_config)
575
+
576
self.input_is_parallel = input_is_parallel
577
self.reduce_results = reduce_results
563
- if params_dtype is None:
564
- params_dtype = torch.get_default_dtype()
565
- self.params_dtype = params_dtype
578
579
# Divide the weight matrix along the last dimension.
580
self.tp_size = get_tensor_model_parallel_world_size()
581
self.input_size_per_partition = divide(input_size, self.tp_size)
570
- self.skip_bias_add = skip_bias_add
571
- if linear_method is None:
572
- linear_method = UnquantizedLinearMethod()
573
- self.linear_method = linear_method
574
- self.linear_method.create_weights(self,
575
- self.input_size_per_partition,
576
- [self.output_size],
577
- self.input_size,
578
- self.output_size,
579
- self.params_dtype,
580
- weight_loader=self.weight_loader)
582
+ self.quant_method.create_weights(self,
583
+ self.input_size_per_partition,
584
+ [self.output_size],
585
+ self.input_size,
586
+ self.output_size,
587
+ self.params_dtype,
588
+ weight_loader=self.weight_loader)
589
590
if not reduce_results and (bias and not skip_bias_add):
591
raise ValueError("When not reduce the results, adding bias to the "
@@ -616,8 +624,7 @@ class RowParallelLinear(torch.nn.Module):
624
input_parallel = splitted_input[tp_rank].contiguous()
625
626
# Matrix multiply.
619
- output_parallel = self.linear_method.apply_weights(
620
- self, input_parallel)
627
+ output_parallel = self.quant_method.apply(self, input_parallel)
628
if self.reduce_results and self.tp_size > 1:
629
output_ = tensor_model_parallel_all_reduce(output_parallel)
630
else:
vllm/model_executor/layers/quantization/__init__.py
+2
-2
@@ -4,7 +4,7 @@ from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4
from vllm.model_executor.layers.quantization.awq import AWQConfig
5
from vllm.model_executor.layers.quantization.base_config import (
6
QuantizationConfig)
7
-from vllm.model_executor.layers.quantization.fp8 import FP8Config
7
+from vllm.model_executor.layers.quantization.fp8 import Fp8Config
8
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
9
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
10
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
12
QUANTIZATION_METHODS = {
13
"aqlm": AQLMConfig,
14
"awq": AWQConfig,
15
- "fp8": FP8Config,
15
+ "fp8": Fp8Config,
16
"gptq": GPTQConfig,
17
"squeezellm": SqueezeLLMConfig,
18
"marlin": MarlinConfig,
vllm/model_executor/layers/quantization/aqlm.py
+8
-5
@@ -9,10 +9,10 @@ import torch.nn.functional as F
9
from torch.nn.parameter import Parameter
10
11
from vllm import _custom_ops as ops
12
-from vllm.model_executor.layers.linear import (LinearMethodBase,
13
- set_weight_attrs)
12
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
13
from vllm.model_executor.layers.quantization.base_config import (
14
QuantizationConfig)
15
+from vllm.model_executor.utils import set_weight_attrs
16
17
18
def get_int_dtype(nbits: int) -> torch.dtype:
@@ -207,8 +207,11 @@ class AQLMConfig(QuantizationConfig):
207
return cls(in_group_size, nbits_per_codebook, num_code_books,
208
out_group_size)
209
210
- def get_linear_method(self) -> "AQLMLinearMethod":
211
- return AQLMLinearMethod(self)
210
+ def get_quant_method(
211
+ self, layer: torch.nn.Module) -> Optional["AQLMLinearMethod"]:
212
+ if isinstance(layer, LinearBase):
213
+ return AQLMLinearMethod(self)
214
+ return None
215
216
def get_scaled_act_names(self) -> List[str]:
217
return []
@@ -321,7 +324,7 @@ class AQLMLinearMethod(LinearMethodBase):
324
layer.register_parameter("scales", scales)
325
set_weight_attrs(scales, extra_weight_attrs)
326
324
- def apply_weights(
327
+ def apply(
328
self,
329
layer: torch.nn.Module,
330
x: torch.Tensor,
vllm/model_executor/layers/quantization/awq.py
+11
-8
@@ -4,10 +4,10 @@ import torch
4
from torch.nn.parameter import Parameter
5
6
from vllm import _custom_ops as ops
7
-from vllm.model_executor.layers.linear import (LinearMethodBase,
8
- set_weight_attrs)
7
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8
from vllm.model_executor.layers.quantization.base_config import (
9
QuantizationConfig)
10
+from vllm.model_executor.utils import set_weight_attrs
11
12
13
class AWQConfig(QuantizationConfig):
@@ -62,8 +62,11 @@ class AWQConfig(QuantizationConfig):
62
zero_point = cls.get_from_keys(config, ["zero_point"])
63
return cls(weight_bits, group_size, zero_point)
64
65
- def get_linear_method(self) -> "AWQLinearMethod":
66
- return AWQLinearMethod(self)
65
+ def get_quant_method(
66
+ self, layer: torch.nn.Module) -> Optional["AWQLinearMethod"]:
67
+ if isinstance(layer, LinearBase):
68
+ return AWQLinearMethod(self)
69
+ return None
70
71
def get_scaled_act_names(self) -> List[str]:
72
return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
@@ -147,10 +150,10 @@ class AWQLinearMethod(LinearMethodBase):
150
layer.register_parameter("scales", scales)
151
set_weight_attrs(scales, extra_weight_attrs)
152
150
- def apply_weights(self,
151
- layer: torch.nn.Module,
152
- x: torch.Tensor,
153
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
153
+ def apply(self,
154
+ layer: torch.nn.Module,
155
+ x: torch.Tensor,
156
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
157
qweight = layer.qweight
158
scales = layer.scales
159
qzeros = layer.qzeros
vllm/model_executor/layers/quantization/base_config.py
+28
-3
@@ -2,8 +2,33 @@ from abc import ABC, abstractmethod
2
from typing import Any, Dict, List
3
4
import torch
5
+from torch import nn
6
6
-from vllm.model_executor.layers.linear import LinearMethodBase
7
+
8
+class QuantizeMethodBase(ABC):
9
+ """Base class for different quantized methods."""
10
+
11
+ @abstractmethod
12
+ def create_weights(self, layer: torch.nn.Module, *weight_args,
13
+ **extra_weight_attrs):
14
+ """Create weights for a layer.
15
+
16
+ The weights will be set as attributes of the layer."""
17
+ raise NotImplementedError
18
+
19
+ @abstractmethod
20
+ def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
21
+ """Apply the weights in layer to the input tensor.
22
+
23
+ Expects create_weights to have been called before on the layer."""
24
+ raise NotImplementedError
25
+
26
+ def process_weights_after_loading(self, layer: nn.Module) -> None:
27
+ """Process the weight after loading.
28
+
29
+ This can be used for example, to transpose weights for computation.
30
+ """
31
+ return
32
33
34
class QuantizationConfig(ABC):
@@ -51,8 +76,8 @@ class QuantizationConfig(ABC):
76
"quantization config.")
77
78
@abstractmethod
54
- def get_linear_method(self) -> LinearMethodBase:
55
- """Get the linear method to use for the quantized linear layer."""
79
+ def get_quant_method(self, layer: torch.nn.Module) -> QuantizeMethodBase:
80
+ """Get the quantize method to use for the quantized layer."""
81
raise NotImplementedError
82
83
@abstractmethod
vllm/model_executor/layers/quantization/fp8.py
+20
-40
@@ -1,16 +1,17 @@
1
-from typing import Any, Dict, List, Optional, Tuple
1
+from typing import Any, Dict, List, Optional
2
3
import torch
4
from torch.nn import Module
5
from torch.nn.parameter import Parameter
6
7
-from vllm.model_executor.layers.linear import (LinearMethodBase,
8
- set_weight_attrs)
7
+from vllm import _custom_ops as ops
8
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
from vllm.model_executor.layers.quantization.base_config import (
10
- QuantizationConfig)
10
+ QuantizationConfig, QuantizeMethodBase)
11
+from vllm.model_executor.utils import set_weight_attrs
12
13
13
-class FP8Config(QuantizationConfig):
14
+class Fp8Config(QuantizationConfig):
15
"""Config class for FP8."""
16
17
@classmethod
@@ -33,11 +34,14 @@ class FP8Config(QuantizationConfig):
34
return []
35
36
@classmethod
36
- def from_config(cls, config: Dict[str, Any]) -> "FP8Config":
37
+ def from_config(cls, config: Dict[str, Any]) -> "Fp8Config":
38
return cls()
39
39
- def get_linear_method(self) -> "Fp8LinearMethod":
40
- return Fp8LinearMethod(self)
40
+ def get_quant_method(
41
+ self, layer: torch.nn.Module) -> Optional["QuantizeMethodBase"]:
42
+ if isinstance(layer, LinearBase):
43
+ return Fp8LinearMethod(self)
44
+ return None
45
46
def get_scaled_act_names(self) -> List[str]:
47
return []
@@ -57,7 +61,7 @@ class Fp8LinearMethod(LinearMethodBase):
61
quant_config: The quantization config.
62
"""
63
60
- def __init__(self, quant_config: FP8Config):
64
+ def __init__(self, quant_config: Fp8Config):
65
self.quant_config = quant_config
66
67
def create_weights(
@@ -86,24 +90,24 @@ class Fp8LinearMethod(LinearMethodBase):
90
layer.register_parameter("weight_scaling_factor", w_scale)
91
92
def process_weights_after_loading(self, layer: Module) -> None:
89
- # Although the linear_method is propagated to all layers,
93
+ # Although the quant_method is propagated to all layers,
94
# only linear layers invoke "create_weights". So we check
95
# whether "weight_scaling_facor" is registered to determine
96
# whether the layer is a linear layer that requires quantization.
97
if not hasattr(layer, "weight_scaling_factor"):
98
return
99
96
- qweight, weight_scale = per_tensor_quantize(layer.weight)
100
+ qweight, weight_scale = ops.scaled_fp8_quant(layer.weight)
101
# torch._scaled_mm requires column-major in the second
102
# input (weight), so we transpose the quantized weight.
103
layer.weight = Parameter(qweight.t(), requires_grad=False)
104
layer.weight_scaling_factor.data.copy_(weight_scale)
105
102
- def apply_weights(self,
103
- layer: torch.nn.Module,
104
- x: torch.Tensor,
105
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
106
- qinput, x_scale = per_tensor_quantize(x)
106
+ def apply(self,
107
+ layer: torch.nn.Module,
108
+ x: torch.Tensor,
109
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
110
+ qinput, x_scale = ops.scaled_fp8_quant(x)
111
output, _ = torch._scaled_mm(
112
qinput,
113
layer.weight,
@@ -113,27 +117,3 @@ class Fp8LinearMethod(LinearMethodBase):
117
bias=bias,
118
)
119
return output
116
-
117
-
118
-def per_tensor_quantize(tensor: torch.Tensor) -> Tuple[torch.Tensor, float]:
119
- """Quantize a tensor using per-tensor static scaling factor.
120
-
121
- Args:
122
- tensor: The input tensor.
123
- """
124
- finfo = torch.finfo(torch.float8_e4m3fn)
125
- # Calculate the scale as dtype max divided by absmax.
126
- # Since .abs() creates a new tensor, we use aminmax to get
127
- # the min and max first and then calculate the absmax.
128
- min_val, max_val = tensor.aminmax()
129
- amax = min_val.abs().max(max_val.abs())
130
- scale = finfo.max / amax.clamp(min=1e-12)
131
- # scale and clamp the tensor to bring it to
132
- # the representative range of float8 data type
133
- # (as default cast is unsaturated)
134
- qweight = (tensor * scale).clamp(min=finfo.min, max=finfo.max)
135
- # Return both float8 data and the inverse scale (as float),
136
- # as both required as inputs to torch._scaled_mm
137
- qweight = qweight.to(torch.float8_e4m3fn)
138
- scale = scale.float().reciprocal()
139
- return qweight, scale
vllm/model_executor/layers/quantization/gptq.py
+11
-8
@@ -7,10 +7,10 @@ import torch
7
from torch.nn.parameter import Parameter
8
9
from vllm import _custom_ops as ops
10
-from vllm.model_executor.layers.linear import (LinearMethodBase,
11
- set_weight_attrs)
10
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11
from vllm.model_executor.layers.quantization.base_config import (
12
QuantizationConfig)
13
+from vllm.model_executor.utils import set_weight_attrs
14
15
16
class GPTQConfig(QuantizationConfig):
@@ -63,8 +63,11 @@ class GPTQConfig(QuantizationConfig):
63
desc_act = cls.get_from_keys(config, ["desc_act"])
64
return cls(weight_bits, group_size, desc_act)
65
66
- def get_linear_method(self) -> "GPTQLinearMethod":
67
- return GPTQLinearMethod(self)
66
+ def get_quant_method(
67
+ self, layer: torch.nn.Module) -> Optional["GPTQLinearMethod"]:
68
+ if isinstance(layer, LinearBase):
69
+ return GPTQLinearMethod(self)
70
+ return None
71
72
def get_scaled_act_names(self) -> List[str]:
73
return []
@@ -194,10 +197,10 @@ class GPTQLinearMethod(LinearMethodBase):
197
198
layer.exllama_state = exllama_state
199
197
- def apply_weights(self,
198
- layer: torch.nn.Module,
199
- x: torch.Tensor,
200
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
200
+ def apply(self,
201
+ layer: torch.nn.Module,
202
+ x: torch.Tensor,
203
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
204
qweight = layer.qweight
205
out_shape = x.shape[:-1] + (qweight.shape[-1], )
206
reshaped_x = x.reshape(-1, x.shape[-1])
vllm/model_executor/layers/quantization/marlin.py
+8
-5
@@ -4,10 +4,10 @@ import torch
4
from torch.nn.parameter import Parameter
5
6
from vllm import _custom_ops as ops
7
-from vllm.model_executor.layers.linear import (LinearMethodBase,
8
- set_weight_attrs)
7
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8
from vllm.model_executor.layers.quantization.base_config import (
9
QuantizationConfig)
10
+from vllm.model_executor.utils import set_weight_attrs
11
12
13
class MarlinConfig(QuantizationConfig):
@@ -72,8 +72,11 @@ class MarlinConfig(QuantizationConfig):
72
group_size = cls.get_from_keys(config, ["group_size"])
73
return cls(group_size)
74
75
- def get_linear_method(self) -> "MarlinLinearMethod":
76
- return MarlinLinearMethod(self)
75
+ def get_quant_method(
76
+ self, layer: torch.nn.Module) -> Optional["MarlinLinearMethod"]:
77
+ if isinstance(layer, LinearBase):
78
+ return MarlinLinearMethod(self)
79
+ return None
80
81
def get_scaled_act_names(self) -> List[str]:
82
return []
@@ -197,7 +200,7 @@ class MarlinLinearMethod(LinearMethodBase):
200
layer.register_parameter("workspace", workspace)
201
set_weight_attrs(workspace, extra_weight_attrs)
202
200
- def apply_weights(
203
+ def apply(
204
self,
205
layer: torch.nn.Module,
206
x: torch.Tensor,
vllm/model_executor/layers/quantization/squeezellm.py
+14
-10
@@ -4,10 +4,10 @@ import torch
4
from torch.nn.parameter import Parameter
5
6
from vllm import _custom_ops as ops
7
-from vllm.model_executor.layers.linear import (LinearMethodBase,
8
- set_weight_attrs)
7
+from vllm.model_executor.layers.linear import LinearBase
8
from vllm.model_executor.layers.quantization.base_config import (
10
- QuantizationConfig)
9
+ QuantizationConfig, QuantizeMethodBase)
10
+from vllm.model_executor.utils import set_weight_attrs
11
from vllm.utils import is_hip
12
13
@@ -51,14 +51,18 @@ class SqueezeLLMConfig(QuantizationConfig):
51
weight_bits = cls.get_from_keys(config, ["wbits"])
52
return cls(weight_bits)
53
54
- def get_linear_method(self) -> "SqueezeLLMLinearMethod":
55
- return SqueezeLLMLinearMethod(self)
54
+ def get_quant_method(
55
+ self,
56
+ layer: torch.nn.Module) -> Optional["SqueezeLLMLinearMethod"]:
57
+ if isinstance(layer, LinearBase):
58
+ return SqueezeLLMLinearMethod(self)
59
+ return
60
61
def get_scaled_act_names(self) -> List[str]:
62
return []
63
64
61
-class SqueezeLLMLinearMethod(LinearMethodBase):
65
+class SqueezeLLMLinearMethod(QuantizeMethodBase):
66
"""Linear method for SqueezeLLM.
67
68
Args:
@@ -112,10 +116,10 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
116
layer.register_parameter("lookup_table", lookup_table)
117
set_weight_attrs(lookup_table, extra_weight_attrs)
118
115
- def apply_weights(self,
116
- layer: torch.nn.Module,
117
- x: torch.Tensor,
118
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
119
+ def apply(self,
120
+ layer: torch.nn.Module,
121
+ x: torch.Tensor,
122
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
123
qweight = layer.qweight
124
lookup_table = layer.lookup_table
125
out_shape = x.shape[:-1] + (qweight.shape[-1], )
vllm/model_executor/model_loader/loader.py
+18
-20
@@ -3,8 +3,7 @@ import copy
3
import glob
4
import os
5
from abc import ABC, abstractmethod
6
-from typing import (TYPE_CHECKING, Any, Dict, Generator, List, Optional, Tuple,
7
- Type)
6
+from typing import Any, Dict, Generator, List, Optional, Tuple, Type
7
8
import torch
9
from torch import nn
@@ -13,6 +12,8 @@ from vllm.config import (VLLM_USE_MODELSCOPE, DeviceConfig, LoadConfig,
12
LoadFormat, LoRAConfig, ModelConfig, ParallelConfig,
13
SchedulerConfig, VisionLanguageConfig)
14
from vllm.logger import init_logger
15
+from vllm.model_executor.layers.quantization.base_config import (
16
+ QuantizationConfig)
17
from vllm.model_executor.model_loader.tensorizer import (
18
TensorizerConfig, is_vllm_serialized_tensorizer, load_with_tensorizer,
19
tensorizer_weights_iterator)
@@ -24,9 +25,6 @@ from vllm.model_executor.model_loader.weight_utils import (
25
pt_weights_iterator, safetensors_weights_iterator)
26
from vllm.model_executor.models.llava import LlavaForConditionalGeneration
27
27
-if TYPE_CHECKING:
28
- from vllm.model_executor.layers.linear import LinearMethodBase
29
-
28
_VISION_MODEL_CLASSES = [
29
LlavaForConditionalGeneration,
30
]
@@ -34,11 +32,10 @@ _VISION_MODEL_CLASSES = [
32
logger = init_logger(__name__)
33
34
37
-def _get_linear_method(
35
+def _get_quantization_config(
36
model_config: ModelConfig,
39
- load_config: LoadConfig) -> Optional["LinearMethodBase"]:
40
- """Get the (maybe quantized) linear method."""
41
- linear_method = None
37
+ load_config: LoadConfig) -> Optional[QuantizationConfig]:
38
+ """Get the quantization config."""
39
if model_config.quantization is not None:
40
quant_config = get_quant_config(model_config, load_config)
41
capability = torch.cuda.get_device_capability()
@@ -55,9 +52,8 @@ def _get_linear_method(
52
f"{model_config.dtype} is not supported for quantization "
53
f"method {model_config.quantization}. Supported dtypes: "
54
f"{supported_dtypes}")
58
-
59
- linear_method = quant_config.get_linear_method()
60
- return linear_method
55
+ return quant_config
56
+ return None
57
58
59
def _get_model_initialization_kwargs(
@@ -85,10 +81,10 @@ def _initialize_model(
81
vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
82
"""Initialize a model with the given configurations."""
83
model_class = get_model_architecture(model_config)[0]
88
- linear_method = _get_linear_method(model_config, load_config)
84
+ quant_config = _get_quantization_config(model_config, load_config)
85
86
return model_class(config=model_config.hf_config,
91
- linear_method=linear_method,
87
+ quant_config=quant_config,
88
**_get_model_initialization_kwargs(
89
model_class, lora_config, vision_language_config))
90
@@ -229,9 +225,11 @@ class DefaultModelLoader(BaseModelLoader):
225
"fall_back_to_pt_during_load",
226
True)), )
227
for _, module in model.named_modules():
232
- linear_method = getattr(module, "linear_method", None)
233
- if linear_method is not None:
234
- linear_method.process_weights_after_loading(module)
228
+ quant_method = getattr(module, "quant_method", None)
229
+ if quant_method is not None:
230
+ quant_method.process_weights_after_loading(module)
231
+ # FIXME: Remove this after Mixtral is updated
232
+ # to use quant_method.
233
if hasattr(module, "process_weights_after_loading"):
234
module.process_weights_after_loading()
235
return model.eval()
@@ -314,11 +312,11 @@ class TensorizerLoader(BaseModelLoader):
312
with set_default_torch_dtype(model_config.dtype):
313
with torch.device(device_config.device):
314
model_class = get_model_architecture(model_config)[0]
317
- linear_method = _get_linear_method(model_config,
318
- self.load_config)
315
+ quant_config = _get_quantization_config(
316
+ model_config, self.load_config)
317
extra_kwargs = _get_model_initialization_kwargs(
318
model_class, lora_config, vision_language_config)
321
- extra_kwargs["linear_method"] = linear_method
319
+ extra_kwargs["quant_config"] = quant_config
320
321
tensorizer_config = copy.copy(self.tensorizer_config)
322
tensorizer_config.model_class = model_class
vllm/model_executor/model_loader/tensorizer.py
+7
-6
@@ -13,7 +13,8 @@ from transformers import PretrainedConfig
13
14
from vllm.config import ModelConfig, ParallelConfig
15
from vllm.logger import init_logger
16
-from vllm.model_executor.layers.linear import LinearMethodBase
16
+from vllm.model_executor.layers.quantization.base_config import (
17
+ QuantizationConfig)
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
19
VocabParallelEmbedding)
20
@@ -251,7 +252,7 @@ class TensorizerAgent:
252
"""
253
254
def __init__(self, tensorizer_config: TensorizerConfig,
254
- linear_method: LinearMethodBase, **extra_kwargs):
255
+ quant_config: QuantizationConfig, **extra_kwargs):
256
if tensorizer_load_fail is not None:
257
raise ImportError(
258
"Tensorizer is not installed. Please install tensorizer "
@@ -262,10 +263,10 @@ class TensorizerAgent:
263
self.tensorizer_args = (
264
self.tensorizer_config._construct_tensorizer_args())
265
self.extra_kwargs = extra_kwargs
265
- if extra_kwargs.get("linear_method", None) is not None:
266
- self.linear_method = extra_kwargs["linear_method"]
266
+ if extra_kwargs.get("quant_config", None) is not None:
267
+ self.quant_config = extra_kwargs["quant_config"]
268
else:
268
- self.linear_method = linear_method
269
+ self.quant_config = quant_config
270
self.model = self._init_model()
271
272
def _init_model(self):
@@ -274,7 +275,7 @@ class TensorizerAgent:
275
with no_init_or_tensor():
276
return self.tensorizer_config.model_class(
277
config=model_args,
277
- linear_method=self.linear_method,
278
+ quant_config=self.quant_config,
279
**self.extra_kwargs)
280
281
def _resize_lora_embeddings(self):
vllm/model_executor/models/baichuan.py
+22
-21
@@ -31,11 +31,12 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
31
get_tensor_model_parallel_world_size)
32
from vllm.model_executor.layers.activation import SiluAndMul
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
-from vllm.model_executor.layers.linear import (LinearMethodBase,
35
- MergedColumnParallelLinear,
34
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
35
QKVParallelLinear,
36
RowParallelLinear)
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
+from vllm.model_executor.layers.quantization.base_config import (
39
+ QuantizationConfig)
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -77,17 +78,17 @@ class BaiChuanMLP(nn.Module):
78
hidden_size: int,
79
intermediate_size: int,
80
hidden_act: str,
80
- linear_method: Optional[LinearMethodBase] = None,
81
+ quant_config: Optional[QuantizationConfig] = None,
82
):
83
super().__init__()
84
self.gate_up_proj = MergedColumnParallelLinear(
85
hidden_size, [intermediate_size] * 2,
86
bias=False,
86
- linear_method=linear_method)
87
+ quant_config=quant_config)
88
self.down_proj = RowParallelLinear(intermediate_size,
89
hidden_size,
90
bias=False,
90
- linear_method=linear_method)
91
+ quant_config=quant_config)
92
if hidden_act != "silu":
93
raise ValueError(f"Unsupported activation: {hidden_act}. "
94
"Only silu is supported for now.")
@@ -110,7 +111,7 @@ class BaiChuanAttention(nn.Module):
111
position_embedding: str,
112
rope_theta: float = 10000,
113
max_position_embeddings: int = 8192,
113
- linear_method: Optional[LinearMethodBase] = None,
114
+ quant_config: Optional[QuantizationConfig] = None,
115
):
116
super().__init__()
117
self.hidden_size = hidden_size
@@ -132,13 +133,13 @@ class BaiChuanAttention(nn.Module):
133
self.total_num_heads,
134
self.total_num_heads,
135
bias=False,
135
- linear_method=linear_method,
136
+ quant_config=quant_config,
137
)
138
self.o_proj = RowParallelLinear(
139
self.total_num_heads * self.head_dim,
140
hidden_size,
141
bias=False,
141
- linear_method=linear_method,
142
+ quant_config=quant_config,
143
)
144
# Create the alibi slopes and slice them.
145
if self.postion_embedding == "ALIBI":
@@ -184,7 +185,7 @@ class BaiChuanDecoderLayer(nn.Module):
185
def __init__(self,
186
config: PretrainedConfig,
187
position_embedding: str,
187
- linear_method: Optional[LinearMethodBase] = None):
188
+ quant_config: Optional[QuantizationConfig] = None):
189
super().__init__()
190
self.hidden_size = config.hidden_size
191
rope_theta = getattr(config, "rope_theta", 10000)
@@ -196,13 +197,13 @@ class BaiChuanDecoderLayer(nn.Module):
197
position_embedding=position_embedding,
198
rope_theta=rope_theta,
199
max_position_embeddings=max_position_embeddings,
199
- linear_method=linear_method,
200
+ quant_config=quant_config,
201
)
202
self.mlp = BaiChuanMLP(
203
hidden_size=self.hidden_size,
204
intermediate_size=config.intermediate_size,
205
hidden_act=config.hidden_act,
205
- linear_method=linear_method,
206
+ quant_config=quant_config,
207
)
208
self.input_layernorm = RMSNorm(config.hidden_size,
209
eps=config.rms_norm_eps)
@@ -243,7 +244,7 @@ class BaiChuanModel(nn.Module):
244
def __init__(self,
245
config: PretrainedConfig,
246
position_embedding: str,
246
- linear_method: Optional[LinearMethodBase] = None):
247
+ quant_config: Optional[QuantizationConfig] = None):
248
super().__init__()
249
self.config = config
250
self.padding_idx = config.pad_token_id
@@ -254,7 +255,7 @@ class BaiChuanModel(nn.Module):
255
config.hidden_size,
256
)
257
self.layers = nn.ModuleList([
257
- BaiChuanDecoderLayer(config, position_embedding, linear_method)
258
+ BaiChuanDecoderLayer(config, position_embedding, quant_config)
259
for _ in range(config.num_hidden_layers)
260
])
261
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -303,13 +304,13 @@ class BaiChuanBaseForCausalLM(nn.Module):
304
self,
305
config,
306
position_embedding: str,
306
- linear_method: Optional[LinearMethodBase] = None,
307
+ quant_config: Optional[QuantizationConfig] = None,
308
lora_config: Optional[LoRAConfig] = None,
309
):
310
super().__init__()
311
self.config = config
311
- self.linear_method = linear_method
312
- self.model = BaiChuanModel(config, position_embedding, linear_method)
312
+ self.quant_config = quant_config
313
+ self.model = BaiChuanModel(config, position_embedding, quant_config)
314
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
315
self.logits_processor = LogitsProcessor(config.vocab_size)
316
self.sampler = Sampler()
@@ -388,13 +389,13 @@ class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
389
def __init__(
390
self,
391
config,
391
- linear_method: Optional[LinearMethodBase] = None,
392
+ quant_config: Optional[QuantizationConfig] = None,
393
lora_config: Optional[LoRAConfig] = None,
394
):
395
if config.hidden_size == 4096: # baichuan2 7b
395
- super().__init__(config, "ROPE", linear_method, lora_config)
396
+ super().__init__(config, "ROPE", quant_config, lora_config)
397
else: # baichuan 13b, baichuan2 13b
397
- super().__init__(config, "ALIBI", linear_method, lora_config)
398
+ super().__init__(config, "ALIBI", quant_config, lora_config)
399
400
401
class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
@@ -403,7 +404,7 @@ class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
404
def __init__(
405
self,
406
config,
406
- linear_method: Optional[LinearMethodBase] = None,
407
+ quant_config: Optional[QuantizationConfig] = None,
408
lora_config: Optional[LoRAConfig] = None,
409
):
409
- super().__init__(config, "ROPE", linear_method, lora_config)
410
+ super().__init__(config, "ROPE", quant_config, lora_config)
vllm/model_executor/models/bloom.py
+17
-16
@@ -28,10 +28,11 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
28
get_tensor_model_parallel_world_size)
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
- LinearMethodBase,
31
QKVParallelLinear,
32
RowParallelLinear)
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
+from vllm.model_executor.layers.quantization.base_config import (
35
+ QuantizationConfig)
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
@@ -70,7 +71,7 @@ class BloomAttention(nn.Module):
71
def __init__(
72
self,
73
config: BloomConfig,
73
- linear_method: Optional[LinearMethodBase] = None,
74
+ quant_config: Optional[QuantizationConfig] = None,
75
):
76
super().__init__()
77
self.hidden_size = config.hidden_size
@@ -87,13 +88,13 @@ class BloomAttention(nn.Module):
88
self.head_dim,
89
self.total_num_heads,
90
bias=True,
90
- linear_method=linear_method,
91
+ quant_config=quant_config,
92
)
93
self.dense = RowParallelLinear(
94
self.hidden_size,
95
self.hidden_size,
96
bias=True,
96
- linear_method=linear_method,
97
+ quant_config=quant_config,
98
)
99
100
# Create the alibi slopes and slice them.
@@ -129,21 +130,21 @@ class BloomMLP(nn.Module):
130
def __init__(
131
self,
132
config: BloomConfig,
132
- linear_method: Optional[LinearMethodBase] = None,
133
+ quant_config: Optional[QuantizationConfig] = None,
134
):
135
super().__init__()
136
hidden_size = config.hidden_size
137
self.dense_h_to_4h = ColumnParallelLinear(
138
hidden_size,
139
4 * hidden_size,
139
- linear_method=linear_method,
140
+ quant_config=quant_config,
141
)
141
- quant_config = getattr(linear_method, "quant_config", None)
142
+ quant_config = getattr(quant_config, "quant_config", None)
143
self.gelu_impl = get_act_fn("gelu", quant_config, 4 * hidden_size)
144
self.dense_4h_to_h = RowParallelLinear(
145
4 * hidden_size,
146
hidden_size,
146
- linear_method=linear_method,
147
+ quant_config=quant_config,
148
)
149
150
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -158,17 +159,17 @@ class BloomBlock(nn.Module):
159
def __init__(
160
self,
161
config: BloomConfig,
161
- linear_method: Optional[LinearMethodBase] = None,
162
+ quant_config: Optional[QuantizationConfig] = None,
163
):
164
super().__init__()
165
hidden_size = config.hidden_size
166
167
self.input_layernorm = nn.LayerNorm(hidden_size,
168
eps=config.layer_norm_epsilon)
168
- self.self_attention = BloomAttention(config, linear_method)
169
+ self.self_attention = BloomAttention(config, quant_config)
170
self.post_attention_layernorm = nn.LayerNorm(
171
hidden_size, eps=config.layer_norm_epsilon)
171
- self.mlp = BloomMLP(config, linear_method)
172
+ self.mlp = BloomMLP(config, quant_config)
173
self.apply_residual_connection_post_layernorm = (
174
config.apply_residual_connection_post_layernorm)
175
@@ -214,7 +215,7 @@ class BloomModel(nn.Module):
215
def __init__(
216
self,
217
config: BloomConfig,
217
- linear_method: Optional[LinearMethodBase] = None,
218
+ quant_config: Optional[QuantizationConfig] = None,
219
):
220
super().__init__()
221
self.embed_dim = config.hidden_size
@@ -229,7 +230,7 @@ class BloomModel(nn.Module):
230
231
# Transformer blocks
232
self.h = nn.ModuleList([
232
- BloomBlock(config, linear_method)
233
+ BloomBlock(config, quant_config)
234
for _ in range(config.num_hidden_layers)
235
])
236
@@ -262,12 +263,12 @@ class BloomForCausalLM(nn.Module):
263
def __init__(
264
self,
265
config: BloomConfig,
265
- linear_method: Optional[LinearMethodBase] = None,
266
+ quant_config: Optional[QuantizationConfig] = None,
267
):
268
super().__init__()
269
self.config = config
269
- self.linear_method = linear_method
270
- self.transformer = BloomModel(config, linear_method)
270
+ self.quant_config = quant_config
271
+ self.transformer = BloomModel(config, quant_config)
272
self.lm_head_weight = self.transformer.word_embeddings.weight
273
self.logits_processor = LogitsProcessor(config.vocab_size)
274
self.sampler = Sampler()
vllm/model_executor/models/chatglm.py
+19
-18
@@ -13,11 +13,12 @@ from vllm.config import LoRAConfig
13
from vllm.distributed import get_tensor_model_parallel_world_size
14
from vllm.model_executor.layers.activation import SiluAndMul
15
from vllm.model_executor.layers.layernorm import RMSNorm
16
-from vllm.model_executor.layers.linear import (LinearMethodBase,
17
- MergedColumnParallelLinear,
16
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
17
QKVParallelLinear,
18
RowParallelLinear)
19
from vllm.model_executor.layers.logits_processor import LogitsProcessor
20
+from vllm.model_executor.layers.quantization.base_config import (
21
+ QuantizationConfig)
22
from vllm.model_executor.layers.rotary_embedding import get_rope
23
from vllm.model_executor.layers.sampler import Sampler
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -33,7 +34,7 @@ class GLMAttention(nn.Module):
34
def __init__(
35
self,
36
config,
36
- linear_method: Optional[LinearMethodBase] = None,
37
+ quant_config: Optional[QuantizationConfig] = None,
38
):
39
super().__init__()
40
self.hidden_size = config.hidden_size
@@ -65,13 +66,13 @@ class GLMAttention(nn.Module):
66
self.total_num_heads,
67
self.total_num_kv_heads,
68
bias=config.add_bias_linear or config.add_qkv_bias,
68
- linear_method=linear_method,
69
+ quant_config=quant_config,
70
)
71
self.dense = RowParallelLinear(
72
self.total_num_heads * self.head_dim,
73
config.hidden_size,
74
bias=config.add_bias_linear,
74
- linear_method=linear_method,
75
+ quant_config=quant_config,
76
)
77
78
# https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
@@ -123,7 +124,7 @@ class GLMMLP(nn.Module):
124
def __init__(
125
self,
126
config,
126
- linear_method: Optional[LinearMethodBase] = None,
127
+ quant_config: Optional[QuantizationConfig] = None,
128
):
129
super().__init__()
130
@@ -134,7 +135,7 @@ class GLMMLP(nn.Module):
135
config.hidden_size,
136
[config.ffn_hidden_size] * 2,
137
bias=config.add_bias_linear,
137
- linear_method=linear_method,
138
+ quant_config=quant_config,
139
)
140
141
self.activation_func = SiluAndMul()
@@ -144,7 +145,7 @@ class GLMMLP(nn.Module):
145
config.ffn_hidden_size,
146
config.hidden_size,
147
bias=config.add_bias_linear,
147
- linear_method=linear_method,
148
+ quant_config=quant_config,
149
)
150
151
def forward(self, hidden_states):
@@ -166,7 +167,7 @@ class GLMBlock(nn.Module):
167
def __init__(
168
self,
169
config,
169
- linear_method: Optional[LinearMethodBase] = None,
170
+ quant_config: Optional[QuantizationConfig] = None,
171
):
172
super().__init__()
173
self.apply_residual_connection_post_layernorm = (
@@ -180,7 +181,7 @@ class GLMBlock(nn.Module):
181
eps=config.layernorm_epsilon)
182
183
# Self attention.
183
- self.self_attention = GLMAttention(config, linear_method)
184
+ self.self_attention = GLMAttention(config, quant_config)
185
self.hidden_dropout = config.hidden_dropout
186
187
# Layernorm on the attention output
@@ -188,7 +189,7 @@ class GLMBlock(nn.Module):
189
config.hidden_size, eps=config.layernorm_epsilon)
190
191
# MLP
191
- self.mlp = GLMMLP(config, linear_method)
192
+ self.mlp = GLMMLP(config, quant_config)
193
194
def forward(
195
self,
@@ -236,7 +237,7 @@ class GLMTransformer(nn.Module):
237
def __init__(
238
self,
239
config,
239
- linear_method: Optional[LinearMethodBase] = None,
240
+ quant_config: Optional[QuantizationConfig] = None,
241
):
242
super().__init__()
243
self.post_layer_norm = config.post_layer_norm
@@ -246,7 +247,7 @@ class GLMTransformer(nn.Module):
247
248
# Transformer layers.
249
self.layers = nn.ModuleList(
249
- [GLMBlock(config, linear_method) for i in range(self.num_layers)])
250
+ [GLMBlock(config, quant_config) for i in range(self.num_layers)])
251
252
if self.post_layer_norm:
253
layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -281,7 +282,7 @@ class ChatGLMModel(nn.Module):
282
def __init__(
283
self,
284
config,
284
- linear_method: Optional[LinearMethodBase] = None,
285
+ quant_config: Optional[QuantizationConfig] = None,
286
):
287
super().__init__()
288
@@ -291,7 +292,7 @@ class ChatGLMModel(nn.Module):
292
self.num_layers = config.num_layers
293
self.multi_query_group_num = config.multi_query_group_num
294
self.kv_channels = config.kv_channels
294
- self.encoder = GLMTransformer(config, linear_method)
295
+ self.encoder = GLMTransformer(config, quant_config)
296
297
self.output_layer = ParallelLMHead(config.padded_vocab_size,
298
config.hidden_size)
@@ -333,13 +334,13 @@ class ChatGLMForCausalLM(nn.Module):
334
def __init__(
335
self,
336
config: ChatGLMConfig,
336
- linear_method: Optional[LinearMethodBase] = None,
337
+ quant_config: Optional[QuantizationConfig] = None,
338
lora_config: Optional[LoRAConfig] = None,
339
):
340
super().__init__()
341
self.config: ChatGLMConfig = config
341
- self.linear_method = linear_method
342
- self.transformer = ChatGLMModel(config, linear_method)
342
+ self.quant_config = quant_config
343
+ self.transformer = ChatGLMModel(config, quant_config)
344
self.lm_head_weight = self.transformer.output_layer.weight
345
self.logits_processor = LogitsProcessor(config.padded_vocab_size)
346
self.sampler = Sampler()
vllm/model_executor/models/commandr.py
+17
-16
@@ -32,11 +32,12 @@ from vllm.attention import Attention, AttentionMetadata
32
from vllm.distributed import (get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size)
34
from vllm.model_executor.layers.activation import SiluAndMul
35
-from vllm.model_executor.layers.linear import (LinearMethodBase,
36
- MergedColumnParallelLinear,
35
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
36
QKVParallelLinear,
37
RowParallelLinear)
38
from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
+from vllm.model_executor.layers.quantization.base_config import (
40
+ QuantizationConfig)
41
from vllm.model_executor.layers.rotary_embedding import get_rope
42
from vllm.model_executor.layers.sampler import Sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -91,7 +92,7 @@ class CohereMLP(nn.Module):
92
def __init__(
93
self,
94
config,
94
- linear_method: Optional[LinearMethodBase] = None,
95
+ quant_config: Optional[QuantizationConfig] = None,
96
):
97
super().__init__()
98
self.config = config
@@ -101,13 +102,13 @@ class CohereMLP(nn.Module):
102
self.hidden_size,
103
[self.intermediate_size] * 2,
104
bias=False,
104
- linear_method=linear_method,
105
+ quant_config=quant_config,
106
)
107
self.down_proj = RowParallelLinear(
108
self.intermediate_size,
109
self.hidden_size,
110
bias=False,
110
- linear_method=linear_method,
111
+ quant_config=quant_config,
112
)
113
self.act_fn = SiluAndMul()
114
@@ -123,7 +124,7 @@ class CohereAttention(nn.Module):
124
def __init__(
125
self,
126
config: CohereConfig,
126
- linear_method: Optional[LinearMethodBase] = None,
127
+ quant_config: Optional[QuantizationConfig] = None,
128
):
129
super().__init__()
130
tp_size = get_tensor_model_parallel_world_size()
@@ -158,13 +159,13 @@ class CohereAttention(nn.Module):
159
self.total_num_heads,
160
self.total_num_kv_heads,
161
bias=False,
161
- linear_method=linear_method,
162
+ quant_config=quant_config,
163
)
164
self.o_proj = RowParallelLinear(
165
self.total_num_heads * self.head_dim,
166
self.hidden_size,
167
bias=False,
167
- linear_method=linear_method,
168
+ quant_config=quant_config,
169
)
170
self.rotary_emb = get_rope(
171
self.head_dim,
@@ -218,13 +219,13 @@ class CohereDecoderLayer(nn.Module):
219
220
def __init__(self,
221
config: CohereConfig,
221
- linear_method: Optional[LinearMethodBase] = None):
222
+ quant_config: Optional[QuantizationConfig] = None):
223
super().__init__()
224
self.hidden_size = config.hidden_size
225
225
- self.self_attn = CohereAttention(config, linear_method=linear_method)
226
+ self.self_attn = CohereAttention(config, quant_config=quant_config)
227
227
- self.mlp = CohereMLP(config, linear_method=linear_method)
228
+ self.mlp = CohereMLP(config, quant_config=quant_config)
229
self.input_layernorm = LayerNorm(param_shape=(config.hidden_size),
230
eps=config.layer_norm_eps)
231
@@ -257,7 +258,7 @@ class CohereModel(nn.Module):
258
def __init__(
259
self,
260
config: CohereConfig,
260
- linear_method: Optional[LinearMethodBase] = None,
261
+ quant_config: Optional[QuantizationConfig] = None,
262
):
263
super().__init__()
264
self.config = config
@@ -265,7 +266,7 @@ class CohereModel(nn.Module):
266
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
267
config.hidden_size)
268
self.layers = nn.ModuleList([
268
- CohereDecoderLayer(config, linear_method=linear_method)
269
+ CohereDecoderLayer(config, quant_config=quant_config)
270
for _ in range(config.num_hidden_layers)
271
])
272
self.norm = LayerNorm(param_shape=(config.hidden_size),
@@ -298,14 +299,14 @@ class CohereForCausalLM(nn.Module):
299
def __init__(
300
self,
301
config: CohereConfig,
301
- linear_method: Optional[LinearMethodBase] = None,
302
+ quant_config: Optional[QuantizationConfig] = None,
303
) -> None:
304
super().__init__()
305
self.config = config
305
- self.linear_method = linear_method
306
+ self.quant_config = quant_config
307
self.logits_processor = LogitsProcessor(config.vocab_size,
308
scale=config.logit_scale)
308
- self.model = CohereModel(config, linear_method)
309
+ self.model = CohereModel(config, quant_config)
310
self.sampler = Sampler()
311
312
@torch.no_grad()
vllm/model_executor/models/dbrx.py
+18
-17
@@ -9,11 +9,12 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
9
get_tensor_model_parallel_world_size,
10
tensor_model_parallel_all_reduce)
11
from vllm.model_executor.layers.fused_moe import fused_moe
12
-from vllm.model_executor.layers.linear import (LinearMethodBase,
13
- QKVParallelLinear,
12
+from vllm.model_executor.layers.linear import (QKVParallelLinear,
13
ReplicatedLinear,
14
RowParallelLinear)
15
from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
+from vllm.model_executor.layers.quantization.base_config import (
17
+ QuantizationConfig)
18
from vllm.model_executor.layers.rotary_embedding import get_rope
19
from vllm.model_executor.layers.sampler import Sampler
20
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -44,7 +45,7 @@ class DbrxRouter(nn.Module):
45
self.num_total_experts,
46
bias=False,
47
params_dtype=params_dtype,
47
- linear_method=None,
48
+ quant_config=None,
49
)
50
51
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -63,7 +64,7 @@ class DbrxExperts(nn.Module):
64
def __init__(
65
self,
66
config: DbrxConfig,
66
- linear_method: Optional[LinearMethodBase] = None,
67
+ quant_config: Optional[QuantizationConfig] = None,
68
params_dtype: Optional[torch.dtype] = None,
69
):
70
super().__init__()
@@ -165,7 +166,7 @@ class DbrxAttention(nn.Module):
166
def __init__(
167
self,
168
config: DbrxConfig,
168
- linear_method: Optional[LinearMethodBase] = None,
169
+ quant_config: Optional[QuantizationConfig] = None,
170
):
171
super().__init__()
172
self.d_model = config.d_model
@@ -183,13 +184,13 @@ class DbrxAttention(nn.Module):
184
self.total_num_heads,
185
self.total_num_kv_heads,
186
bias=False,
186
- linear_method=linear_method,
187
+ quant_config=quant_config,
188
)
189
self.out_proj = RowParallelLinear(
190
self.d_model,
191
self.d_model,
192
bias=False,
192
- linear_method=linear_method,
193
+ quant_config=quant_config,
194
)
195
self.rotary_emb = get_rope(
196
self.head_dim,
@@ -244,11 +245,11 @@ class DbrxFusedNormAttention(nn.Module):
245
def __init__(
246
self,
247
config: DbrxConfig,
247
- linear_method: Optional[LinearMethodBase] = None,
248
+ quant_config: Optional[QuantizationConfig] = None,
249
):
250
super().__init__()
251
self.d_model = config.d_model
251
- self.attn = DbrxAttention(config, linear_method)
252
+ self.attn = DbrxAttention(config, quant_config)
253
self.norm_1 = nn.LayerNorm(self.d_model)
254
self.norm_2 = nn.LayerNorm(self.d_model)
255
@@ -278,11 +279,11 @@ class DbrxBlock(nn.Module):
279
def __init__(
280
self,
281
config: DbrxConfig,
281
- linear_method: Optional[LinearMethodBase] = None,
282
+ quant_config: Optional[QuantizationConfig] = None,
283
):
284
super().__init__()
284
- self.norm_attn_norm = DbrxFusedNormAttention(config, linear_method)
285
- self.ffn = DbrxExperts(config, linear_method)
285
+ self.norm_attn_norm = DbrxFusedNormAttention(config, quant_config)
286
+ self.ffn = DbrxExperts(config, quant_config)
287
288
def forward(
289
self,
@@ -307,7 +308,7 @@ class DbrxModel(nn.Module):
308
def __init__(
309
self,
310
config: DbrxConfig,
310
- linear_method: Optional[LinearMethodBase] = None,
311
+ quant_config: Optional[QuantizationConfig] = None,
312
):
313
super().__init__()
314
self.wte = VocabParallelEmbedding(
@@ -315,7 +316,7 @@ class DbrxModel(nn.Module):
316
config.d_model,
317
)
318
self.blocks = nn.ModuleList(
318
- [DbrxBlock(config, linear_method) for _ in range(config.n_layers)])
319
+ [DbrxBlock(config, quant_config) for _ in range(config.n_layers)])
320
self.norm_f = nn.LayerNorm(config.d_model, eps=1e-5)
321
for module in self.modules():
322
if hasattr(module, "bias") and isinstance(module.bias,
@@ -348,13 +349,13 @@ class DbrxForCausalLM(nn.Module):
349
def __init__(
350
self,
351
config: DbrxConfig,
351
- linear_method: Optional[LinearMethodBase] = None,
352
+ quant_config: Optional[QuantizationConfig] = None,
353
):
354
super().__init__()
355
self.config = config
355
- self.linear_method = linear_method
356
+ self.quant_config = quant_config
357
self.unpadded_vocab_size = config.vocab_size
357
- self.transformer = DbrxModel(config, linear_method)
358
+ self.transformer = DbrxModel(config, quant_config)
359
self.lm_head = ParallelLMHead(
360
config.vocab_size,
361
config.d_model,
vllm/model_executor/models/decilm.py
+4
-3
@@ -29,7 +29,8 @@ import torch
29
from transformers import PretrainedConfig
30
31
from vllm.config import LoRAConfig
32
-from vllm.model_executor.layers.linear import LinearMethodBase
32
+from vllm.model_executor.layers.quantization.base_config import (
33
+ QuantizationConfig)
34
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35
from vllm.model_executor.models.llama import LlamaForCausalLM
36
@@ -55,13 +56,13 @@ class DeciLMForCausalLM(LlamaForCausalLM):
56
def __init__(
57
self,
58
config: Optional[PretrainedConfig] = None,
58
- linear_method: Optional[LinearMethodBase] = None,
59
+ quant_config: Optional[QuantizationConfig] = None,
60
lora_config: Optional[LoRAConfig] = None,
61
) -> None:
62
config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
63
delattr(config, "num_key_value_heads_per_layer")
64
super().__init__(config=config,
64
- linear_method=linear_method,
65
+ quant_config=quant_config,
66
lora_config=lora_config)
67
68
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
vllm/model_executor/models/deepseek.py
+22
-23
@@ -34,12 +34,13 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.fused_moe import fused_moe
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
-from vllm.model_executor.layers.linear import (LinearMethodBase,
38
- MergedColumnParallelLinear,
37
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
38
QKVParallelLinear,
39
ReplicatedLinear,
40
RowParallelLinear)
41
from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
+from vllm.model_executor.layers.quantization.base_config import (
43
+ QuantizationConfig)
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
from vllm.model_executor.layers.sampler import Sampler
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -56,18 +57,18 @@ class DeepseekMLP(nn.Module):
57
hidden_size: int,
58
intermediate_size: int,
59
hidden_act: str,
59
- linear_method: Optional[LinearMethodBase] = None,
60
+ quant_config: Optional[QuantizationConfig] = None,
61
reduce_results: bool = True,
62
) -> None:
63
super().__init__()
64
self.gate_up_proj = MergedColumnParallelLinear(
65
hidden_size, [intermediate_size] * 2,
66
bias=False,
66
- linear_method=linear_method)
67
+ quant_config=quant_config)
68
self.down_proj = RowParallelLinear(intermediate_size,
69
hidden_size,
70
bias=False,
70
- linear_method=linear_method,
71
+ quant_config=quant_config,
72
reduce_results=reduce_results)
73
if hidden_act != "silu":
74
raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -86,7 +87,7 @@ class DeepseekMoE(nn.Module):
87
def __init__(
88
self,
89
config: PretrainedConfig,
89
- linear_method: Optional[LinearMethodBase] = None,
90
+ quant_config: Optional[QuantizationConfig] = None,
91
):
92
super().__init__()
93
self.config = config
@@ -103,7 +104,7 @@ class DeepseekMoE(nn.Module):
104
DeepseekMLP(hidden_size=config.hidden_size,
105
intermediate_size=config.moe_intermediate_size,
106
hidden_act=config.hidden_act,
106
- linear_method=linear_method,
107
+ quant_config=quant_config,
108
reduce_results=False)
109
for idx in range(self.n_routed_experts)
110
])
@@ -112,7 +113,7 @@ class DeepseekMoE(nn.Module):
113
self.gate = ReplicatedLinear(config.hidden_size,
114
self.n_routed_experts,
115
bias=False,
115
- linear_method=None)
116
+ quant_config=None)
117
118
if config.n_shared_experts is not None:
119
intermediate_size = (config.moe_intermediate_size *
@@ -121,7 +122,7 @@ class DeepseekMoE(nn.Module):
122
hidden_size=config.hidden_size,
123
intermediate_size=intermediate_size,
124
hidden_act=config.hidden_act,
124
- linear_method=linear_method,
125
+ quant_config=quant_config,
126
reduce_results=False,
127
)
128
@@ -177,7 +178,7 @@ class DeepseekAttention(nn.Module):
178
rope_theta: float = 10000,
179
rope_scaling: Optional[Dict[str, Any]] = None,
180
max_position_embeddings: int = 8192,
180
- linear_method: Optional[LinearMethodBase] = None,
181
+ quant_config: Optional[QuantizationConfig] = None,
182
) -> None:
183
super().__init__()
184
self.hidden_size = hidden_size
@@ -208,14 +209,14 @@ class DeepseekAttention(nn.Module):
209
self.total_num_heads,
210
self.total_num_kv_heads,
211
bias=False,
211
- linear_method=linear_method,
212
+ quant_config=quant_config,
213
)
214
215
self.o_proj = RowParallelLinear(
216
self.total_num_heads * self.head_dim,
217
hidden_size,
218
bias=False,
218
- linear_method=linear_method,
219
+ quant_config=quant_config,
220
)
221
222
self.rotary_emb = get_rope(
@@ -251,7 +252,7 @@ class DeepseekDecoderLayer(nn.Module):
252
self,
253
config: PretrainedConfig,
254
layer_idx: int,
254
- linear_method: Optional[LinearMethodBase] = None,
255
+ quant_config: Optional[QuantizationConfig] = None,
256
) -> None:
257
super().__init__()
258
self.hidden_size = config.hidden_size
@@ -266,18 +267,18 @@ class DeepseekDecoderLayer(nn.Module):
267
rope_theta=rope_theta,
268
rope_scaling=rope_scaling,
269
max_position_embeddings=max_position_embeddings,
269
- linear_method=linear_method,
270
+ quant_config=quant_config,
271
)
272
if (config.n_routed_experts is not None
273
and layer_idx >= config.first_k_dense_replace
274
and layer_idx % config.moe_layer_freq == 0):
274
- self.mlp = DeepseekMoE(config=config, linear_method=linear_method)
275
+ self.mlp = DeepseekMoE(config=config, quant_config=quant_config)
276
else:
277
self.mlp = DeepseekMLP(
278
hidden_size=config.hidden_size,
279
intermediate_size=config.intermediate_size,
280
hidden_act=config.hidden_act,
280
- linear_method=linear_method,
281
+ quant_config=quant_config,
282
)
283
self.input_layernorm = RMSNorm(config.hidden_size,
284
eps=config.rms_norm_eps)
@@ -320,7 +321,7 @@ class DeepseekModel(nn.Module):
321
def __init__(
322
self,
323
config: PretrainedConfig,
323
- linear_method: Optional[LinearMethodBase] = None,
324
+ quant_config: Optional[QuantizationConfig] = None,
325
) -> None:
326
super().__init__()
327
self.padding_idx = config.pad_token_id
@@ -331,9 +332,7 @@ class DeepseekModel(nn.Module):
332
config.hidden_size,
333
)
334
self.layers = nn.ModuleList([
334
- DeepseekDecoderLayer(config,
335
- layer_idx,
336
- linear_method=linear_method)
335
+ DeepseekDecoderLayer(config, layer_idx, quant_config=quant_config)
336
for layer_idx in range(config.num_hidden_layers)
337
])
338
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -361,12 +360,12 @@ class DeepseekForCausalLM(nn.Module):
360
def __init__(
361
self,
362
config: PretrainedConfig,
364
- linear_method: Optional[LinearMethodBase] = None,
363
+ quant_config: Optional[QuantizationConfig] = None,
364
) -> None:
365
super().__init__()
366
self.config = config
368
- self.linear_method = linear_method
369
- self.model = DeepseekModel(config, linear_method)
367
+ self.quant_config = quant_config
368
+ self.model = DeepseekModel(config, quant_config)
369
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
370
self.logits_processor = LogitsProcessor(config.vocab_size)
371
self.sampler = Sampler()
vllm/model_executor/models/falcon.py
+17
-16
@@ -32,10 +32,11 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
32
tensor_model_parallel_all_reduce)
33
from vllm.model_executor.layers.activation import get_act_fn
34
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35
- LinearMethodBase,
35
QKVParallelLinear,
36
RowParallelLinear)
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
+from vllm.model_executor.layers.quantization.base_config import (
39
+ QuantizationConfig)
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -76,7 +77,7 @@ class FalconAttention(nn.Module):
77
def __init__(
78
self,
79
config: FalconConfig,
79
- linear_method: Optional[LinearMethodBase] = None,
80
+ quant_config: Optional[QuantizationConfig] = None,
81
):
82
super().__init__()
83
@@ -115,7 +116,7 @@ class FalconAttention(nn.Module):
116
self.total_num_kv_heads,
117
bias=config.bias,
118
skip_bias_add=True,
118
- linear_method=linear_method,
119
+ quant_config=quant_config,
120
)
121
self.q_size = self.num_heads * self.head_dim
122
self.kv_size = self.num_kv_heads * self.head_dim
@@ -129,7 +130,7 @@ class FalconAttention(nn.Module):
130
self.hidden_size,
131
bias=config.bias,
132
skip_bias_add=True,
132
- linear_method=linear_method,
133
+ quant_config=quant_config,
134
reduce_results=self.reduce_row_parallel_results)
135
136
self.use_rotary = config.rotary
@@ -192,7 +193,7 @@ class FalconMLP(nn.Module):
193
def __init__(
194
self,
195
config: FalconConfig,
195
- linear_method: Optional[LinearMethodBase] = None,
196
+ quant_config: Optional[QuantizationConfig] = None,
197
):
198
super().__init__()
199
hidden_size = config.hidden_size
@@ -201,8 +202,8 @@ class FalconMLP(nn.Module):
202
4 * hidden_size,
203
bias=config.bias,
204
skip_bias_add=True,
204
- linear_method=linear_method)
205
- quant_config = getattr(linear_method, "quant_config", None)
205
+ quant_config=quant_config)
206
+ quant_config = getattr(quant_config, "quant_config", None)
207
self.act = get_act_fn("gelu", quant_config, 4 * hidden_size)
208
self.reduce_row_parallel_results = not (config.new_decoder_architecture
209
or config.parallel_attn)
@@ -212,7 +213,7 @@ class FalconMLP(nn.Module):
213
bias=config.bias,
214
skip_bias_add=True,
215
reduce_results=self.reduce_row_parallel_results,
215
- linear_method=linear_method)
216
+ quant_config=quant_config)
217
218
def forward(self, x: torch.Tensor) -> torch.Tensor:
219
# NOTE(zhuohan): Following huggingface, we do not fuse bias add here.
@@ -229,13 +230,13 @@ class FalconDecoderLayer(nn.Module):
230
def __init__(
231
self,
232
config: FalconConfig,
232
- linear_method: Optional[LinearMethodBase] = None,
233
+ quant_config: Optional[QuantizationConfig] = None,
234
):
235
super().__init__()
236
hidden_size = config.hidden_size
237
self.num_heads = config.num_attention_heads
237
- self.self_attention = FalconAttention(config, linear_method)
238
- self.mlp = FalconMLP(config, linear_method)
238
+ self.self_attention = FalconAttention(config, quant_config)
239
+ self.mlp = FalconMLP(config, quant_config)
240
self.config = config
241
242
if config.new_decoder_architecture:
@@ -311,7 +312,7 @@ class FalconModel(nn.Module):
312
def __init__(
313
self,
314
config: FalconConfig,
314
- linear_method: Optional[LinearMethodBase] = None,
315
+ quant_config: Optional[QuantizationConfig] = None,
316
):
317
super().__init__()
318
self.config = config
@@ -327,7 +328,7 @@ class FalconModel(nn.Module):
328
329
# Transformer blocks
330
self.h = nn.ModuleList([
330
- FalconDecoderLayer(config, linear_method)
331
+ FalconDecoderLayer(config, quant_config)
332
for _ in range(config.num_hidden_layers)
333
])
334
@@ -359,12 +360,12 @@ class FalconForCausalLM(nn.Module):
360
def __init__(
361
self,
362
config: FalconConfig,
362
- linear_method: Optional[LinearMethodBase] = None,
363
+ quant_config: Optional[QuantizationConfig] = None,
364
):
365
super().__init__()
366
self.config = config
366
- self.linear_method = linear_method
367
- self.transformer = FalconModel(config, linear_method)
367
+ self.quant_config = quant_config
368
+ self.transformer = FalconModel(config, quant_config)
369
self.lm_head_weight = self.transformer.word_embeddings.weight
370
self.logits_processor = LogitsProcessor(config.vocab_size)
371
self.sampler = Sampler()
vllm/model_executor/models/gemma.py
+17
-16
@@ -27,11 +27,12 @@ from vllm.distributed import get_tensor_model_parallel_world_size
27
from vllm.logger import init_logger
28
from vllm.model_executor.layers.activation import GeluAndMul
29
from vllm.model_executor.layers.layernorm import RMSNorm
30
-from vllm.model_executor.layers.linear import (LinearMethodBase,
31
- MergedColumnParallelLinear,
30
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31
QKVParallelLinear,
32
RowParallelLinear)
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
+from vllm.model_executor.layers.quantization.base_config import (
35
+ QuantizationConfig)
36
from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -77,17 +78,17 @@ class GemmaMLP(nn.Module):
78
intermediate_size: int,
79
hidden_act: Optional[str] = None,
80
hidden_activation: Optional[str] = None,
80
- linear_method: Optional[LinearMethodBase] = None,
81
+ quant_config: Optional[QuantizationConfig] = None,
82
) -> None:
83
super().__init__()
84
self.gate_up_proj = MergedColumnParallelLinear(
85
hidden_size, [intermediate_size] * 2,
86
bias=False,
86
- linear_method=linear_method)
87
+ quant_config=quant_config)
88
self.down_proj = RowParallelLinear(intermediate_size,
89
hidden_size,
90
bias=False,
90
- linear_method=linear_method)
91
+ quant_config=quant_config)
92
self.act_fn = _get_gemma_act_fn(hidden_act, hidden_activation)
93
94
def forward(self, x):
@@ -106,7 +107,7 @@ class GemmaAttention(nn.Module):
107
head_dim: int,
108
max_position_embeddings: int = 8192,
109
rope_theta: float = 10000,
109
- linear_method: Optional[LinearMethodBase] = None) -> None:
110
+ quant_config: Optional[QuantizationConfig] = None) -> None:
111
super().__init__()
112
self.hidden_size = hidden_size
113
tp_size = get_tensor_model_parallel_world_size()
@@ -135,13 +136,13 @@ class GemmaAttention(nn.Module):
136
self.total_num_heads,
137
self.total_num_kv_heads,
138
bias=False,
138
- linear_method=linear_method,
139
+ quant_config=quant_config,
140
)
141
self.o_proj = RowParallelLinear(
142
self.total_num_heads * self.head_dim,
143
hidden_size,
144
bias=False,
144
- linear_method=linear_method,
145
+ quant_config=quant_config,
146
)
147
148
self.rotary_emb = get_rope(
@@ -176,7 +177,7 @@ class GemmaDecoderLayer(nn.Module):
177
def __init__(
178
self,
179
config: GemmaConfig,
179
- linear_method: Optional[LinearMethodBase] = None,
180
+ quant_config: Optional[QuantizationConfig] = None,
181
) -> None:
182
super().__init__()
183
self.hidden_size = config.hidden_size
@@ -187,14 +188,14 @@ class GemmaDecoderLayer(nn.Module):
188
head_dim=config.head_dim,
189
max_position_embeddings=config.max_position_embeddings,
190
rope_theta=config.rope_theta,
190
- linear_method=linear_method,
191
+ quant_config=quant_config,
192
)
193
self.mlp = GemmaMLP(
194
hidden_size=self.hidden_size,
195
intermediate_size=config.intermediate_size,
196
hidden_act=config.hidden_act,
197
hidden_activation=getattr(config, "hidden_activation", None),
197
- linear_method=linear_method,
198
+ quant_config=quant_config,
199
)
200
self.input_layernorm = RMSNorm(config.hidden_size,
201
eps=config.rms_norm_eps)
@@ -235,7 +236,7 @@ class GemmaModel(nn.Module):
236
def __init__(
237
self,
238
config: GemmaConfig,
238
- linear_method: Optional[LinearMethodBase] = None,
239
+ quant_config: Optional[QuantizationConfig] = None,
240
) -> None:
241
super().__init__()
242
self.config = config
@@ -245,7 +246,7 @@ class GemmaModel(nn.Module):
246
config.hidden_size,
247
)
248
self.layers = nn.ModuleList([
248
- GemmaDecoderLayer(config, linear_method)
249
+ GemmaDecoderLayer(config, quant_config)
250
for _ in range(config.num_hidden_layers)
251
])
252
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -308,14 +309,14 @@ class GemmaForCausalLM(nn.Module):
309
def __init__(
310
self,
311
config: GemmaConfig,
311
- linear_method: Optional[LinearMethodBase] = None,
312
+ quant_config: Optional[QuantizationConfig] = None,
313
lora_config: Optional[LoRAConfig] = None,
314
) -> None:
315
del lora_config # Unused.
316
super().__init__()
317
self.config = config
317
- self.linear_method = linear_method
318
- self.model = GemmaModel(config, linear_method)
318
+ self.quant_config = quant_config
319
+ self.model = GemmaModel(config, quant_config)
320
self.logits_processor = LogitsProcessor(config.vocab_size)
321
self.sampler = Sampler()
322
vllm/model_executor/models/gpt2.py
+17
-16
@@ -27,10 +27,11 @@ from vllm.attention import Attention, AttentionMetadata
27
from vllm.distributed import get_tensor_model_parallel_world_size
28
from vllm.model_executor.layers.activation import get_act_fn
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
- LinearMethodBase,
30
QKVParallelLinear,
31
RowParallelLinear)
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
+from vllm.model_executor.layers.quantization.base_config import (
34
+ QuantizationConfig)
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
@@ -44,7 +45,7 @@ class GPT2Attention(nn.Module):
45
def __init__(
46
self,
47
config: GPT2Config,
47
- linear_method: Optional[LinearMethodBase] = None,
48
+ quant_config: Optional[QuantizationConfig] = None,
49
):
50
super().__init__()
51
self.hidden_size = config.hidden_size
@@ -61,13 +62,13 @@ class GPT2Attention(nn.Module):
62
self.head_dim,
63
total_num_heads,
64
bias=True,
64
- linear_method=linear_method,
65
+ quant_config=quant_config,
66
)
67
self.c_proj = RowParallelLinear(
68
self.hidden_size,
69
self.hidden_size,
70
bias=True,
70
- linear_method=linear_method,
71
+ quant_config=quant_config,
72
)
73
self.attn = Attention(self.num_heads, self.head_dim, scale=self.scale)
74
@@ -90,7 +91,7 @@ class GPT2MLP(nn.Module):
91
self,
92
intermediate_size: int,
93
config: GPT2Config,
93
- linear_method: Optional[LinearMethodBase] = None,
94
+ quant_config: Optional[QuantizationConfig] = None,
95
):
96
super().__init__()
97
hidden_size = config.hidden_size
@@ -98,15 +99,15 @@ class GPT2MLP(nn.Module):
99
hidden_size,
100
intermediate_size,
101
bias=True,
101
- linear_method=linear_method,
102
+ quant_config=quant_config,
103
)
104
self.c_proj = RowParallelLinear(
105
intermediate_size,
106
hidden_size,
107
bias=True,
107
- linear_method=linear_method,
108
+ quant_config=quant_config,
109
)
109
- quant_config = getattr(linear_method, "quant_config", None)
110
+ quant_config = getattr(quant_config, "quant_config", None)
111
self.act = get_act_fn(config.activation_function, quant_config,
112
intermediate_size)
113
@@ -122,7 +123,7 @@ class GPT2Block(nn.Module):
123
def __init__(
124
self,
125
config: GPT2Config,
125
- linear_method: Optional[LinearMethodBase] = None,
126
+ quant_config: Optional[QuantizationConfig] = None,
127
):
128
super().__init__()
129
hidden_size = config.hidden_size
@@ -130,9 +131,9 @@ class GPT2Block(nn.Module):
131
hidden_size)
132
133
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
133
- self.attn = GPT2Attention(config, linear_method)
134
+ self.attn = GPT2Attention(config, quant_config)
135
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
135
- self.mlp = GPT2MLP(inner_dim, config, linear_method)
136
+ self.mlp = GPT2MLP(inner_dim, config, quant_config)
137
138
def forward(
139
self,
@@ -163,7 +164,7 @@ class GPT2Model(nn.Module):
164
def __init__(
165
self,
166
config: GPT2Config,
166
- linear_method: Optional[LinearMethodBase] = None,
167
+ quant_config: Optional[QuantizationConfig] = None,
168
):
169
super().__init__()
170
self.config = config
@@ -174,7 +175,7 @@ class GPT2Model(nn.Module):
175
self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
176
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
177
self.h = nn.ModuleList([
177
- GPT2Block(config, linear_method)
178
+ GPT2Block(config, quant_config)
179
for _ in range(config.num_hidden_layers)
180
])
181
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -203,12 +204,12 @@ class GPT2LMHeadModel(nn.Module):
204
def __init__(
205
self,
206
config: GPT2Config,
206
- linear_method: Optional[LinearMethodBase] = None,
207
+ quant_config: Optional[QuantizationConfig] = None,
208
):
209
super().__init__()
210
self.config = config
210
- self.linear_method = linear_method
211
- self.transformer = GPT2Model(config, linear_method)
211
+ self.quant_config = quant_config
212
+ self.transformer = GPT2Model(config, quant_config)
213
self.lm_head_weight = self.transformer.wte.weight
214
self.logits_processor = LogitsProcessor(config.vocab_size)
215
self.sampler = Sampler()
vllm/model_executor/models/gpt_bigcode.py
+17
-16
@@ -28,10 +28,11 @@ from vllm.attention import Attention, AttentionMetadata
28
from vllm.distributed import get_tensor_model_parallel_world_size
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
- LinearMethodBase,
31
QKVParallelLinear,
32
RowParallelLinear)
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
+from vllm.model_executor.layers.quantization.base_config import (
35
+ QuantizationConfig)
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
@@ -45,7 +46,7 @@ class GPTBigCodeAttention(nn.Module):
46
def __init__(
47
self,
48
config: GPTBigCodeConfig,
48
- linear_method: Optional[LinearMethodBase] = None,
49
+ quant_config: Optional[QuantizationConfig] = None,
50
):
51
super().__init__()
52
self.hidden_size = config.hidden_size
@@ -72,14 +73,14 @@ class GPTBigCodeAttention(nn.Module):
73
total_num_heads,
74
total_num_kv_heads,
75
bias=True,
75
- linear_method=linear_method,
76
+ quant_config=quant_config,
77
)
78
79
self.c_proj = RowParallelLinear(
80
self.hidden_size,
81
self.hidden_size,
82
bias=True,
82
- linear_method=linear_method,
83
+ quant_config=quant_config,
84
)
85
self.attn = Attention(self.num_heads,
86
self.head_dim,
@@ -111,7 +112,7 @@ class GPTBigMLP(nn.Module):
112
self,
113
intermediate_size: int,
114
config: GPTBigCodeConfig,
114
- linear_method: Optional[LinearMethodBase] = None,
115
+ quant_config: Optional[QuantizationConfig] = None,
116
):
117
super().__init__()
118
hidden_size = config.hidden_size
@@ -119,15 +120,15 @@ class GPTBigMLP(nn.Module):
120
hidden_size,
121
intermediate_size,
122
bias=True,
122
- linear_method=linear_method,
123
+ quant_config=quant_config,
124
)
125
self.c_proj = RowParallelLinear(
126
intermediate_size,
127
hidden_size,
128
bias=True,
128
- linear_method=linear_method,
129
+ quant_config=quant_config,
130
)
130
- quant_config = getattr(linear_method, "quant_config", None)
131
+ quant_config = getattr(quant_config, "quant_config", None)
132
self.act = get_act_fn(config.activation_function, quant_config,
133
intermediate_size)
134
@@ -143,7 +144,7 @@ class GPTBigCodeBlock(nn.Module):
144
def __init__(
145
self,
146
config: GPTBigCodeConfig,
146
- linear_method: Optional[LinearMethodBase] = None,
147
+ quant_config: Optional[QuantizationConfig] = None,
148
):
149
super().__init__()
150
hidden_size = config.hidden_size
@@ -151,9 +152,9 @@ class GPTBigCodeBlock(nn.Module):
152
hidden_size)
153
154
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
154
- self.attn = GPTBigCodeAttention(config, linear_method)
155
+ self.attn = GPTBigCodeAttention(config, quant_config)
156
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
156
- self.mlp = GPTBigMLP(inner_dim, config, linear_method)
157
+ self.mlp = GPTBigMLP(inner_dim, config, quant_config)
158
159
def forward(
160
self,
@@ -184,7 +185,7 @@ class GPTBigCodeModel(nn.Module):
185
def __init__(
186
self,
187
config: GPTBigCodeConfig,
187
- linear_method: Optional[LinearMethodBase] = None,
188
+ quant_config: Optional[QuantizationConfig] = None,
189
):
190
super().__init__()
191
self.config = config
@@ -195,7 +196,7 @@ class GPTBigCodeModel(nn.Module):
196
self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
197
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
198
self.h = nn.ModuleList([
198
- GPTBigCodeBlock(config, linear_method)
199
+ GPTBigCodeBlock(config, quant_config)
200
for _ in range(config.num_hidden_layers)
201
])
202
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -224,12 +225,12 @@ class GPTBigCodeForCausalLM(nn.Module):
225
def __init__(
226
self,
227
config: GPTBigCodeConfig,
227
- linear_method: Optional[LinearMethodBase] = None,
228
+ quant_config: Optional[QuantizationConfig] = None,
229
):
230
super().__init__()
231
self.config = config
231
- self.linear_method = linear_method
232
- self.transformer = GPTBigCodeModel(config, linear_method)
232
+ self.quant_config = quant_config
233
+ self.transformer = GPTBigCodeModel(config, quant_config)
234
self.lm_head_weight = self.transformer.wte.weight
235
self.logits_processor = LogitsProcessor(config.vocab_size)
236
self.sampler = Sampler()
vllm/model_executor/models/gpt_j.py
+17
-16
@@ -26,10 +26,11 @@ from vllm.attention import Attention, AttentionMetadata
26
from vllm.distributed import get_tensor_model_parallel_world_size
27
from vllm.model_executor.layers.activation import get_act_fn
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
- LinearMethodBase,
29
QKVParallelLinear,
30
RowParallelLinear)
31
from vllm.model_executor.layers.logits_processor import LogitsProcessor
32
+from vllm.model_executor.layers.quantization.base_config import (
33
+ QuantizationConfig)
34
from vllm.model_executor.layers.rotary_embedding import get_rope
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -44,7 +45,7 @@ class GPTJAttention(nn.Module):
45
def __init__(
46
self,
47
config: GPTJConfig,
47
- linear_method: Optional[LinearMethodBase] = None,
48
+ quant_config: Optional[QuantizationConfig] = None,
49
):
50
super().__init__()
51
self.total_num_heads = config.num_attention_heads
@@ -56,13 +57,13 @@ class GPTJAttention(nn.Module):
57
self.head_size,
58
self.total_num_heads,
59
bias=False,
59
- linear_method=linear_method,
60
+ quant_config=quant_config,
61
)
62
self.out_proj = RowParallelLinear(
63
config.hidden_size,
64
config.hidden_size,
65
bias=False,
65
- linear_method=linear_method,
66
+ quant_config=quant_config,
67
)
68
69
tp_world_size = get_tensor_model_parallel_world_size()
@@ -105,21 +106,21 @@ class GPTJMLP(nn.Module):
106
self,
107
intermediate_size: int,
108
config: GPTJConfig,
108
- linear_method: Optional[LinearMethodBase] = None,
109
+ quant_config: Optional[QuantizationConfig] = None,
110
):
111
super().__init__()
112
hidden_size = config.n_embd
113
self.fc_in = ColumnParallelLinear(
114
hidden_size,
115
intermediate_size,
115
- linear_method=linear_method,
116
+ quant_config=quant_config,
117
)
118
self.fc_out = RowParallelLinear(
119
intermediate_size,
120
hidden_size,
120
- linear_method=linear_method,
121
+ quant_config=quant_config,
122
)
122
- quant_config = getattr(linear_method, "quant_config", None)
123
+ quant_config = getattr(quant_config, "quant_config", None)
124
self.act = get_act_fn(config.activation_function, quant_config,
125
intermediate_size)
126
@@ -135,14 +136,14 @@ class GPTJBlock(nn.Module):
136
def __init__(
137
self,
138
config: GPTJConfig,
138
- linear_method: Optional[LinearMethodBase] = None,
139
+ quant_config: Optional[QuantizationConfig] = None,
140
):
141
super().__init__()
142
inner_dim = (4 * config.n_embd
143
if config.n_inner is None else config.n_inner)
144
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
144
- self.attn = GPTJAttention(config, linear_method)
145
- self.mlp = GPTJMLP(inner_dim, config, linear_method)
145
+ self.attn = GPTJAttention(config, quant_config)
146
+ self.mlp = GPTJMLP(inner_dim, config, quant_config)
147
148
def forward(
149
self,
@@ -169,7 +170,7 @@ class GPTJModel(nn.Module):
170
def __init__(
171
self,
172
config: GPTJConfig,
172
- linear_method: Optional[LinearMethodBase] = None,
173
+ quant_config: Optional[QuantizationConfig] = None,
174
):
175
super().__init__()
176
self.config = config
@@ -179,7 +180,7 @@ class GPTJModel(nn.Module):
180
self.embed_dim,
181
)
182
self.h = nn.ModuleList(
182
- [GPTJBlock(config, linear_method) for _ in range(config.n_layer)])
183
+ [GPTJBlock(config, quant_config) for _ in range(config.n_layer)])
184
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
185
186
def forward(
@@ -207,13 +208,13 @@ class GPTJForCausalLM(nn.Module):
208
def __init__(
209
self,
210
config: GPTJConfig,
210
- linear_method: Optional[LinearMethodBase] = None,
211
+ quant_config: Optional[QuantizationConfig] = None,
212
):
213
super().__init__()
214
self.config = config
214
- self.linear_method = linear_method
215
+ self.quant_config = quant_config
216
assert not config.tie_word_embeddings
216
- self.transformer = GPTJModel(config, linear_method)
217
+ self.transformer = GPTJModel(config, quant_config)
218
self.lm_head = ParallelLMHead(
219
config.vocab_size,
220
config.n_embd,
vllm/model_executor/models/gpt_neox.py
+17
-16
@@ -26,10 +26,11 @@ from vllm.attention import Attention, AttentionMetadata
26
from vllm.distributed import get_tensor_model_parallel_world_size
27
from vllm.model_executor.layers.activation import get_act_fn
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
- LinearMethodBase,
29
QKVParallelLinear,
30
RowParallelLinear)
31
from vllm.model_executor.layers.logits_processor import LogitsProcessor
32
+from vllm.model_executor.layers.quantization.base_config import (
33
+ QuantizationConfig)
34
from vllm.model_executor.layers.rotary_embedding import get_rope
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -44,7 +45,7 @@ class GPTNeoXAttention(nn.Module):
45
def __init__(
46
self,
47
config: GPTNeoXConfig,
47
- linear_method: Optional[LinearMethodBase] = None,
48
+ quant_config: Optional[QuantizationConfig] = None,
49
):
50
super().__init__()
51
self.total_num_heads = config.num_attention_heads
@@ -63,13 +64,13 @@ class GPTNeoXAttention(nn.Module):
64
self.head_size,
65
self.total_num_heads,
66
bias=self.bias,
66
- linear_method=linear_method,
67
+ quant_config=quant_config,
68
)
69
self.dense = RowParallelLinear(
70
config.hidden_size,
71
config.hidden_size,
72
bias=self.bias,
72
- linear_method=linear_method,
73
+ quant_config=quant_config,
74
)
75
scaling = self.head_size**-0.5
76
rotary_dim = int(self.head_size * config.rotary_pct)
@@ -105,20 +106,20 @@ class GPTNeoXMLP(nn.Module):
106
def __init__(
107
self,
108
config: GPTNeoXConfig,
108
- linear_method: Optional[LinearMethodBase] = None,
109
+ quant_config: Optional[QuantizationConfig] = None,
110
):
111
super().__init__()
112
self.dense_h_to_4h = ColumnParallelLinear(
113
config.hidden_size,
114
config.intermediate_size,
114
- linear_method=linear_method,
115
+ quant_config=quant_config,
116
)
117
self.dense_4h_to_h = RowParallelLinear(
118
config.intermediate_size,
119
config.hidden_size,
119
- linear_method=linear_method,
120
+ quant_config=quant_config,
121
)
121
- quant_config = getattr(linear_method, "quant_config", None)
122
+ quant_config = getattr(quant_config, "quant_config", None)
123
self.act = get_act_fn(config.hidden_act, quant_config,
124
config.intermediate_size)
125
@@ -134,7 +135,7 @@ class GPTNeoXLayer(nn.Module):
135
def __init__(
136
self,
137
config: GPTNeoXConfig,
137
- linear_method: Optional[LinearMethodBase] = None,
138
+ quant_config: Optional[QuantizationConfig] = None,
139
):
140
super().__init__()
141
self.use_parallel_residual = config.use_parallel_residual
@@ -142,8 +143,8 @@ class GPTNeoXLayer(nn.Module):
143
eps=config.layer_norm_eps)
144
self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
145
eps=config.layer_norm_eps)
145
- self.attention = GPTNeoXAttention(config, linear_method)
146
- self.mlp = GPTNeoXMLP(config, linear_method)
146
+ self.attention = GPTNeoXAttention(config, quant_config)
147
+ self.mlp = GPTNeoXMLP(config, quant_config)
148
149
def forward(
150
self,
@@ -182,7 +183,7 @@ class GPTNeoXModel(nn.Module):
183
def __init__(
184
self,
185
config: GPTNeoXConfig,
185
- linear_method: Optional[LinearMethodBase] = None,
186
+ quant_config: Optional[QuantizationConfig] = None,
187
):
188
super().__init__()
189
self.config = config
@@ -192,7 +193,7 @@ class GPTNeoXModel(nn.Module):
193
config.hidden_size,
194
)
195
self.layers = nn.ModuleList([
195
- GPTNeoXLayer(config, linear_method)
196
+ GPTNeoXLayer(config, quant_config)
197
for _ in range(config.num_hidden_layers)
198
])
199
self.final_layer_norm = nn.LayerNorm(config.hidden_size,
@@ -223,12 +224,12 @@ class GPTNeoXForCausalLM(nn.Module):
224
def __init__(
225
self,
226
config,
226
- linear_method: Optional[LinearMethodBase] = None,
227
+ quant_config: Optional[QuantizationConfig] = None,
228
):
229
super().__init__()
230
self.config = config
230
- self.linear_method = linear_method
231
- self.gpt_neox = GPTNeoXModel(config, linear_method)
231
+ self.quant_config = quant_config
232
+ self.gpt_neox = GPTNeoXModel(config, quant_config)
233
self.embed_out = ParallelLMHead(
234
config.vocab_size,
235
config.hidden_size,
vllm/model_executor/models/internlm2.py
+17
-16
@@ -9,11 +9,12 @@ from vllm.attention import Attention, AttentionMetadata
9
from vllm.distributed import get_tensor_model_parallel_world_size
10
from vllm.model_executor.layers.activation import SiluAndMul
11
from vllm.model_executor.layers.layernorm import RMSNorm
12
-from vllm.model_executor.layers.linear import (LinearMethodBase,
13
- MergedColumnParallelLinear,
12
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
13
QKVParallelLinear,
14
RowParallelLinear)
15
from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
+from vllm.model_executor.layers.quantization.base_config import (
17
+ QuantizationConfig)
18
from vllm.model_executor.layers.rotary_embedding import get_rope
19
from vllm.model_executor.layers.sampler import Sampler
20
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -30,17 +31,17 @@ class InternLM2MLP(nn.Module):
31
hidden_size: int,
32
intermediate_size: int,
33
hidden_act: str,
33
- linear_method: Optional[LinearMethodBase] = None,
34
+ quant_config: Optional[QuantizationConfig] = None,
35
) -> None:
36
super().__init__()
37
self.gate_up_proj = MergedColumnParallelLinear(
38
hidden_size, [intermediate_size] * 2,
39
bias=False,
39
- linear_method=linear_method)
40
+ quant_config=quant_config)
41
self.w2 = RowParallelLinear(intermediate_size,
42
hidden_size,
43
bias=False,
43
- linear_method=linear_method)
44
+ quant_config=quant_config)
45
if hidden_act != "silu":
46
raise ValueError(f"Unsupported activation: {hidden_act}. "
47
"Only silu is supported for now.")
@@ -63,7 +64,7 @@ class InternLM2Attention(nn.Module):
64
rope_theta: float = 10000,
65
rope_scaling: Optional[Dict[str, Any]] = None,
66
max_position_embeddings: int = 8192,
66
- linear_method: Optional[LinearMethodBase] = None,
67
+ quant_config: Optional[QuantizationConfig] = None,
68
) -> None:
69
super().__init__()
70
self.hidden_size = hidden_size
@@ -94,13 +95,13 @@ class InternLM2Attention(nn.Module):
95
self.total_num_heads,
96
self.total_num_kv_heads,
97
bias=False,
97
- linear_method=linear_method,
98
+ quant_config=quant_config,
99
)
100
self.wo = RowParallelLinear(
101
self.total_num_heads * self.head_dim,
102
hidden_size,
103
bias=False,
103
- linear_method=linear_method,
104
+ quant_config=quant_config,
105
)
106
107
self.rotary_emb = get_rope(
@@ -135,7 +136,7 @@ class InternLMDecoderLayer(nn.Module):
136
def __init__(
137
self,
138
config: PretrainedConfig,
138
- linear_method: Optional[LinearMethodBase] = None,
139
+ quant_config: Optional[QuantizationConfig] = None,
140
) -> None:
141
super().__init__()
142
self.hidden_size = config.hidden_size
@@ -150,13 +151,13 @@ class InternLMDecoderLayer(nn.Module):
151
rope_theta=rope_theta,
152
rope_scaling=rope_scaling,
153
max_position_embeddings=max_position_embeddings,
153
- linear_method=linear_method,
154
+ quant_config=quant_config,
155
)
156
self.feed_forward = InternLM2MLP(
157
hidden_size=self.hidden_size,
158
intermediate_size=config.intermediate_size,
159
hidden_act=config.hidden_act,
159
- linear_method=linear_method,
160
+ quant_config=quant_config,
161
)
162
self.attention_norm = RMSNorm(config.hidden_size,
163
eps=config.rms_norm_eps)
@@ -195,7 +196,7 @@ class InternLM2Model(nn.Module):
196
def __init__(
197
self,
198
config: PretrainedConfig,
198
- linear_method: Optional[LinearMethodBase] = None,
199
+ quant_config: Optional[QuantizationConfig] = None,
200
) -> None:
201
super().__init__()
202
self.config = config
@@ -206,7 +207,7 @@ class InternLM2Model(nn.Module):
207
config.hidden_size,
208
)
209
self.layers = nn.ModuleList([
209
- InternLMDecoderLayer(config, linear_method)
210
+ InternLMDecoderLayer(config, quant_config)
211
for _ in range(config.num_hidden_layers)
212
])
213
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -238,12 +239,12 @@ class InternLM2ForCausalLM(nn.Module):
239
def __init__(
240
self,
241
config: PretrainedConfig,
241
- linear_method: Optional[LinearMethodBase] = None,
242
+ quant_config: Optional[QuantizationConfig] = None,
243
) -> None:
244
super().__init__()
245
self.config = config
245
- self.linear_method = linear_method
246
- self.model = InternLM2Model(config, linear_method)
246
+ self.quant_config = quant_config
247
+ self.model = InternLM2Model(config, quant_config)
248
self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
249
self.logits_processor = LogitsProcessor(config.vocab_size)
250
self.sampler = Sampler()
vllm/model_executor/models/jais.py
+17
-16
@@ -29,10 +29,11 @@ from vllm.attention import Attention, AttentionMetadata
29
from vllm.distributed import (get_tensor_model_parallel_rank,
30
get_tensor_model_parallel_world_size)
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32
- LinearMethodBase,
32
QKVParallelLinear,
33
RowParallelLinear)
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
+from vllm.model_executor.layers.quantization.base_config import (
36
+ QuantizationConfig)
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
@@ -68,7 +69,7 @@ class JAISAttention(nn.Module):
69
def __init__(
70
self,
71
config: JAISConfig,
71
- linear_method: Optional[LinearMethodBase] = None,
72
+ quant_config: Optional[QuantizationConfig] = None,
73
):
74
super().__init__()
75
self.hidden_size = config.hidden_size
@@ -88,13 +89,13 @@ class JAISAttention(nn.Module):
89
self.head_dim,
90
total_num_heads,
91
bias=True,
91
- linear_method=linear_method,
92
+ quant_config=quant_config,
93
)
94
self.c_proj = RowParallelLinear(
95
self.hidden_size,
96
self.hidden_size,
97
bias=True,
97
- linear_method=linear_method,
98
+ quant_config=quant_config,
99
)
100
101
tp_rank = get_tensor_model_parallel_rank()
@@ -128,7 +129,7 @@ class JAISMLP(nn.Module):
129
self,
130
intermediate_size: int,
131
config: JAISConfig,
131
- linear_method: Optional[LinearMethodBase] = None,
132
+ quant_config: Optional[QuantizationConfig] = None,
133
):
134
super().__init__()
135
hidden_size = config.hidden_size
@@ -137,19 +138,19 @@ class JAISMLP(nn.Module):
138
hidden_size,
139
intermediate_size,
140
bias=True,
140
- linear_method=linear_method,
141
+ quant_config=quant_config,
142
)
143
self.c_fc2 = (ColumnParallelLinear(
144
hidden_size,
145
intermediate_size,
146
bias=True,
146
- linear_method=linear_method,
147
+ quant_config=quant_config,
148
) if self.swiglu else None)
149
self.c_proj = RowParallelLinear(
150
intermediate_size,
151
hidden_size,
152
bias=True,
152
- linear_method=linear_method,
153
+ quant_config=quant_config,
154
)
155
156
self.act = SwiGLUActivation()
@@ -169,7 +170,7 @@ class JAISBlock(nn.Module):
170
def __init__(
171
self,
172
config: JAISConfig,
172
- linear_method: Optional[LinearMethodBase] = None,
173
+ quant_config: Optional[QuantizationConfig] = None,
174
):
175
super().__init__()
176
hidden_size = config.hidden_size
@@ -177,9 +178,9 @@ class JAISBlock(nn.Module):
178
hidden_size)
179
180
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
180
- self.attn = JAISAttention(config, linear_method)
181
+ self.attn = JAISAttention(config, quant_config)
182
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
182
- self.mlp = JAISMLP(inner_dim, config, linear_method)
183
+ self.mlp = JAISMLP(inner_dim, config, quant_config)
184
185
def forward(
186
self,
@@ -210,7 +211,7 @@ class JAISModel(nn.Module):
211
def __init__(
212
self,
213
config: JAISConfig,
213
- linear_method: Optional[LinearMethodBase] = None,
214
+ quant_config: Optional[QuantizationConfig] = None,
215
):
216
super().__init__()
217
self.config = config
@@ -227,7 +228,7 @@ class JAISModel(nn.Module):
228
else:
229
self.embeddings_scale = config.mup_embeddings_scale
230
self.h = nn.ModuleList([
230
- JAISBlock(config, linear_method)
231
+ JAISBlock(config, quant_config)
232
for _ in range(config.num_hidden_layers)
233
])
234
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -261,12 +262,12 @@ class JAISLMHeadModel(nn.Module):
262
def __init__(
263
self,
264
config: JAISConfig,
264
- linear_method: Optional[LinearMethodBase] = None,
265
+ quant_config: Optional[QuantizationConfig] = None,
266
):
267
super().__init__()
268
self.config = config
268
- self.linear_method = linear_method
269
- self.transformer = JAISModel(config, linear_method)
269
+ self.quant_config = quant_config
270
+ self.transformer = JAISModel(config, quant_config)
271
self.lm_head_weight = self.transformer.wte.weight
272
if hasattr(config, "width_scale"):
273
self.output_logits_scale = config.width_scale
vllm/model_executor/models/llama.py
+16
-16
@@ -33,11 +33,12 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size)
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.layernorm import RMSNorm
36
-from vllm.model_executor.layers.linear import (LinearMethodBase,
37
- MergedColumnParallelLinear,
36
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37
QKVParallelLinear,
38
RowParallelLinear)
39
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
+from vllm.model_executor.layers.quantization.base_config import (
41
+ QuantizationConfig)
42
from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import Sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -56,17 +57,17 @@ class LlamaMLP(nn.Module):
57
hidden_size: int,
58
intermediate_size: int,
59
hidden_act: str,
59
- linear_method: Optional[LinearMethodBase] = None,
60
+ quant_config: Optional[QKVParallelLinear] = None,
61
) -> None:
62
super().__init__()
63
self.gate_up_proj = MergedColumnParallelLinear(
64
hidden_size, [intermediate_size] * 2,
65
bias=False,
65
- linear_method=linear_method)
66
+ quant_config=quant_config)
67
self.down_proj = RowParallelLinear(intermediate_size,
68
hidden_size,
69
bias=False,
69
- linear_method=linear_method)
70
+ quant_config=quant_config)
71
if hidden_act != "silu":
72
raise ValueError(f"Unsupported activation: {hidden_act}. "
73
"Only silu is supported for now.")
@@ -89,7 +90,7 @@ class LlamaAttention(nn.Module):
90
rope_theta: float = 10000,
91
rope_scaling: Optional[Dict[str, Any]] = None,
92
max_position_embeddings: int = 8192,
92
- linear_method: Optional[LinearMethodBase] = None,
93
+ quant_config: Optional[QuantizationConfig] = None,
94
bias: bool = False,
95
sliding_window: Optional[int] = None,
96
) -> None:
@@ -131,13 +132,13 @@ class LlamaAttention(nn.Module):
132
self.total_num_heads,
133
self.total_num_kv_heads,
134
bias=bias,
134
- linear_method=linear_method,
135
+ quant_config=quant_config,
136
)
137
self.o_proj = RowParallelLinear(
138
self.total_num_heads * self.head_dim,
139
hidden_size,
140
bias=bias,
140
- linear_method=linear_method,
141
+ quant_config=quant_config,
142
)
143
144
self.rotary_emb = get_rope(
@@ -174,7 +175,7 @@ class LlamaDecoderLayer(nn.Module):
175
def __init__(
176
self,
177
config: LlamaConfig,
177
- linear_method: Optional[LinearMethodBase] = None,
178
+ quant_config: Optional[QuantizationConfig] = None,
179
) -> None:
180
super().__init__()
181
self.hidden_size = config.hidden_size
@@ -199,7 +200,7 @@ class LlamaDecoderLayer(nn.Module):
200
rope_theta=rope_theta,
201
rope_scaling=rope_scaling,
202
max_position_embeddings=max_position_embeddings,
202
- linear_method=linear_method,
203
+ quant_config=quant_config,
204
bias=attention_bias,
205
sliding_window=sliding_window,
206
)
@@ -207,7 +208,7 @@ class LlamaDecoderLayer(nn.Module):
208
hidden_size=self.hidden_size,
209
intermediate_size=config.intermediate_size,
210
hidden_act=config.hidden_act,
210
- linear_method=linear_method,
211
+ quant_config=quant_config,
212
)
213
self.input_layernorm = RMSNorm(config.hidden_size,
214
eps=config.rms_norm_eps)
@@ -248,7 +249,7 @@ class LlamaModel(nn.Module):
249
def __init__(
250
self,
251
config: LlamaConfig,
251
- linear_method: Optional[LinearMethodBase] = None,
252
+ quant_config: Optional[QuantizationConfig] = None,
253
lora_config: Optional[LoRAConfig] = None,
254
) -> None:
255
super().__init__()
@@ -264,7 +265,7 @@ class LlamaModel(nn.Module):
265
org_num_embeddings=config.vocab_size,
266
)
267
self.layers = nn.ModuleList([
267
- LlamaDecoderLayer(config, linear_method)
268
+ LlamaDecoderLayer(config, quant_config)
269
for _ in range(config.num_hidden_layers)
270
])
271
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -329,13 +330,12 @@ class LlamaForCausalLM(nn.Module):
330
def __init__(
331
self,
332
config: LlamaConfig,
332
- linear_method: Optional[LinearMethodBase] = None,
333
+ quant_config: Optional[QuantizationConfig] = None,
334
lora_config: Optional[LoRAConfig] = None,
335
) -> None:
336
super().__init__()
337
self.config = config
337
- self.linear_method = linear_method
338
- self.model = LlamaModel(config, linear_method, lora_config=lora_config)
338
+ self.model = LlamaModel(config, quant_config, lora_config=lora_config)
339
self.unpadded_vocab_size = config.vocab_size
340
if lora_config:
341
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/llava.py
+5
-4
@@ -9,8 +9,9 @@ from transformers import CLIPVisionModel, LlavaConfig
9
from vllm.attention import AttentionMetadata
10
from vllm.config import VisionLanguageConfig
11
from vllm.model_executor.layers.activation import get_act_fn
12
-from vllm.model_executor.layers.linear import LinearMethodBase
12
from vllm.model_executor.layers.logits_processor import LogitsProcessor
13
+from vllm.model_executor.layers.quantization.base_config import (
14
+ QuantizationConfig)
15
from vllm.model_executor.layers.sampler import Sampler
16
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
17
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -61,7 +62,7 @@ class LlavaForConditionalGeneration(nn.Module):
62
def __init__(self,
63
config: "LlavaConfig",
64
vision_language_config: VisionLanguageConfig,
64
- linear_method: Optional["LinearMethodBase"] = None) -> None:
65
+ quant_config: Optional["QuantizationConfig"] = None) -> None:
66
super().__init__()
67
self.config = config
68
@@ -83,8 +84,8 @@ class LlavaForConditionalGeneration(nn.Module):
84
text_hidden_size=config.text_config.hidden_size,
85
projector_hidden_act=config.projector_hidden_act)
86
86
- self.linear_method = linear_method
87
- self.language_model = LlamaModel(config.text_config, linear_method)
87
+ self.quant_config = quant_config
88
+ self.language_model = LlamaModel(config.text_config, quant_config)
89
self.unpadded_vocab_size = config.text_config.vocab_size
90
self.lm_head = ParallelLMHead(
91
self.unpadded_vocab_size,
vllm/model_executor/models/minicpm.py
+18
-17
@@ -35,12 +35,13 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
35
from vllm.model_executor.layers.activation import SiluAndMul
36
from vllm.model_executor.layers.fused_moe import fused_moe
37
from vllm.model_executor.layers.layernorm import RMSNorm
38
-from vllm.model_executor.layers.linear import (LinearMethodBase,
39
- MergedColumnParallelLinear,
38
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39
QKVParallelLinear,
40
ReplicatedLinear,
41
RowParallelLinear)
42
from vllm.model_executor.layers.logits_processor import LogitsProcessor
43
+from vllm.model_executor.layers.quantization.base_config import (
44
+ QuantizationConfig)
45
from vllm.model_executor.layers.rotary_embedding import get_rope
46
from vllm.model_executor.layers.sampler import Sampler
47
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -84,7 +85,7 @@ class MiniCPMMoE(nn.Module):
85
self.num_total_experts,
86
bias=False,
87
params_dtype=self.params_dtype,
87
- linear_method=None)
88
+ quant_config=None)
89
90
self.ws = nn.Parameter(
91
torch.empty(self.num_total_experts,
@@ -147,17 +148,17 @@ class MiniCPMMLP(nn.Module):
148
hidden_size: int,
149
intermediate_size: int,
150
hidden_act: str,
150
- linear_method: Optional[LinearMethodBase] = None,
151
+ quant_config: Optional[QuantizationConfig] = None,
152
) -> None:
153
super().__init__()
154
self.gate_up_proj = MergedColumnParallelLinear(
155
hidden_size, [intermediate_size] * 2,
156
bias=False,
156
- linear_method=linear_method)
157
+ quant_config=quant_config)
158
self.down_proj = RowParallelLinear(intermediate_size,
159
hidden_size,
160
bias=False,
160
- linear_method=linear_method)
161
+ quant_config=quant_config)
162
if hidden_act != "silu":
163
raise ValueError(f"Unsupported activation: {hidden_act}. "
164
"Only silu is supported for now.")
@@ -180,7 +181,7 @@ class MiniCPMAttention(nn.Module):
181
rope_theta: float = 10000,
182
rope_scaling: Optional[Dict[str, Any]] = None,
183
max_position_embeddings: int = 8192,
183
- linear_method: Optional[LinearMethodBase] = None,
184
+ quant_config: Optional[QuantizationConfig] = None,
185
) -> None:
186
super().__init__()
187
self.hidden_size = hidden_size
@@ -211,13 +212,13 @@ class MiniCPMAttention(nn.Module):
212
self.total_num_heads,
213
self.total_num_kv_heads,
214
bias=False,
214
- linear_method=linear_method,
215
+ quant_config=quant_config,
216
)
217
self.o_proj = RowParallelLinear(
218
self.total_num_heads * self.head_dim,
219
hidden_size,
220
bias=False,
220
- linear_method=linear_method,
221
+ quant_config=quant_config,
222
)
223
224
self.rotary_emb = get_rope(
@@ -258,7 +259,7 @@ class MiniCPMDecoderLayer(nn.Module):
259
def __init__(
260
self,
261
config,
261
- linear_method: Optional[LinearMethodBase] = None,
262
+ quant_config: Optional[QuantizationConfig] = None,
263
) -> None:
264
super().__init__()
265
self.config = config
@@ -274,7 +275,7 @@ class MiniCPMDecoderLayer(nn.Module):
275
rope_theta=rope_theta,
276
rope_scaling=rope_scaling,
277
max_position_embeddings=max_position_embeddings,
277
- linear_method=linear_method,
278
+ quant_config=quant_config,
279
)
280
self.num_experts = getattr(self.config, "num_experts", 0)
281
if self.num_experts == 0:
@@ -282,7 +283,7 @@ class MiniCPMDecoderLayer(nn.Module):
283
hidden_size=self.hidden_size,
284
intermediate_size=config.intermediate_size,
285
hidden_act=config.hidden_act,
285
- linear_method=linear_method,
286
+ quant_config=quant_config,
287
)
288
else:
289
self.mlp = MiniCPMMoE(num_experts=config.num_experts,
@@ -329,7 +330,7 @@ class MiniCPMModel(nn.Module):
330
def __init__(
331
self,
332
config,
332
- linear_method: Optional[LinearMethodBase] = None,
333
+ quant_config: Optional[QuantizationConfig] = None,
334
lora_config: Optional[LoRAConfig] = None,
335
) -> None:
336
super().__init__()
@@ -345,7 +346,7 @@ class MiniCPMModel(nn.Module):
346
org_num_embeddings=config.vocab_size,
347
)
348
self.layers = nn.ModuleList([
348
- MiniCPMDecoderLayer(config, linear_method)
349
+ MiniCPMDecoderLayer(config, quant_config)
350
for _ in range(config.num_hidden_layers)
351
])
352
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -412,15 +413,15 @@ class MiniCPMForCausalLM(nn.Module):
413
def __init__(
414
self,
415
config,
415
- linear_method: Optional[LinearMethodBase] = None,
416
+ quant_config: Optional[QuantizationConfig] = None,
417
lora_config: Optional[LoRAConfig] = None,
418
) -> None:
419
super().__init__()
420
self.config = config
421
self.num_experts = getattr(self.config, "num_experts", 0)
421
- self.linear_method = linear_method
422
+ self.quant_config = quant_config
423
self.model = MiniCPMModel(config,
423
- linear_method,
424
+ quant_config,
425
lora_config=lora_config)
426
unpadded_vocab_size = config.vocab_size
427
if lora_config:
vllm/model_executor/models/mixtral.py
+22
-22
@@ -27,6 +27,7 @@ import torch
27
from torch import nn
28
from transformers import MixtralConfig
29
30
+from vllm import _custom_ops as ops
31
from vllm.attention import Attention, AttentionMetadata
32
from vllm.config import LoRAConfig
33
from vllm.distributed import (get_tensor_model_parallel_rank,
@@ -34,13 +35,13 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
35
tensor_model_parallel_all_reduce)
36
from vllm.model_executor.layers.fused_moe import fused_moe
37
from vllm.model_executor.layers.layernorm import RMSNorm
37
-from vllm.model_executor.layers.linear import (LinearMethodBase,
38
- QKVParallelLinear,
38
+from vllm.model_executor.layers.linear import (QKVParallelLinear,
39
ReplicatedLinear,
40
RowParallelLinear)
41
from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
-from vllm.model_executor.layers.quantization.fp8 import (Fp8LinearMethod,
43
- per_tensor_quantize)
42
+from vllm.model_executor.layers.quantization.base_config import (
43
+ QuantizationConfig)
44
+from vllm.model_executor.layers.quantization.fp8 import Fp8Config
45
from vllm.model_executor.layers.rotary_embedding import get_rope
46
from vllm.model_executor.layers.sampler import Sampler
47
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -69,7 +70,7 @@ class MixtralMoE(nn.Module):
70
intermediate_size: int,
71
params_dtype: Optional[torch.dtype] = None,
72
tp_size: Optional[int] = None,
72
- linear_method: Optional[LinearMethodBase] = None,
73
+ quant_config: Optional[QuantizationConfig] = None,
74
):
75
super().__init__()
76
self.tp_size = tp_size or get_tensor_model_parallel_world_size()
@@ -79,7 +80,7 @@ class MixtralMoE(nn.Module):
80
self.intermediate_size = intermediate_size // self.tp_size
81
# FIXME(pcmoritz): Make this more general to support different
82
# quantization schemes
82
- self.use_fp8 = isinstance(linear_method, Fp8LinearMethod)
83
+ self.use_fp8 = isinstance(quant_config, Fp8Config)
84
85
if params_dtype is None:
86
params_dtype = torch.get_default_dtype()
@@ -89,7 +90,7 @@ class MixtralMoE(nn.Module):
90
self.num_total_experts,
91
bias=False,
92
params_dtype=self.params_dtype,
92
- linear_method=None)
93
+ quant_config=None)
94
95
self.ws = nn.Parameter(
96
torch.empty(self.num_total_experts,
@@ -140,10 +141,10 @@ class MixtralMoE(nn.Module):
141
ws = torch.empty_like(self.ws.data, dtype=torch.float8_e4m3fn)
142
w2s = torch.empty_like(self.w2s.data, dtype=torch.float8_e4m3fn)
143
for expert in range(self.num_total_experts):
143
- ws[expert, :, :], self.ws_scale[expert] = per_tensor_quantize(
144
+ ws[expert, :, :], self.ws_scale[expert] = ops.scaled_fp8_quant(
145
self.ws.data[expert, :, :])
146
w2s[expert, :, :], self.w2s_scale[
146
- expert] = per_tensor_quantize(self.w2s.data[expert, :, :])
147
+ expert] = ops.scaled_fp8_quant(self.w2s.data[expert, :, :])
148
self.ws = nn.Parameter(ws, requires_grad=False)
149
self.w2s = nn.Parameter(w2s, requires_grad=False)
150
@@ -178,7 +179,7 @@ class MixtralAttention(nn.Module):
179
num_kv_heads: int,
180
max_position: int = 4096 * 32,
181
rope_theta: float = 10000,
181
- linear_method: Optional[LinearMethodBase] = None,
182
+ quant_config: Optional[QuantizationConfig] = None,
183
sliding_window: Optional[int] = None) -> None:
184
super().__init__()
185
self.hidden_size = hidden_size
@@ -203,12 +204,12 @@ class MixtralAttention(nn.Module):
204
self.rope_theta = rope_theta
205
self.sliding_window = sliding_window
206
206
- if isinstance(linear_method, Fp8LinearMethod):
207
+ if isinstance(quant_config, Fp8Config):
208
print_warning_once(
209
"For Mixtral FP8 quantization, we currently do not quantize "
210
"the attention layers until their FP8 performance is improved."
211
)
211
- linear_method = None
212
+ quant_config = None
213
214
self.qkv_proj = QKVParallelLinear(
215
hidden_size,
@@ -216,13 +217,13 @@ class MixtralAttention(nn.Module):
217
self.total_num_heads,
218
self.total_num_kv_heads,
219
bias=False,
219
- linear_method=linear_method,
220
+ quant_config=quant_config,
221
)
222
self.o_proj = RowParallelLinear(
223
self.total_num_heads * self.head_dim,
224
hidden_size,
225
bias=False,
225
- linear_method=linear_method,
226
+ quant_config=quant_config,
227
)
228
self.rotary_emb = get_rope(
229
self.head_dim,
@@ -259,7 +260,7 @@ class MixtralDecoderLayer(nn.Module):
260
def __init__(
261
self,
262
config: MixtralConfig,
262
- linear_method: Optional[LinearMethodBase] = None,
263
+ quant_config: Optional[QuantizationConfig] = None,
264
) -> None:
265
super().__init__()
266
self.hidden_size = config.hidden_size
@@ -272,13 +273,13 @@ class MixtralDecoderLayer(nn.Module):
273
num_kv_heads=config.num_key_value_heads,
274
rope_theta=rope_theta,
275
sliding_window=config.sliding_window,
275
- linear_method=linear_method)
276
+ quant_config=quant_config)
277
self.block_sparse_moe = MixtralMoE(
278
num_experts=config.num_local_experts,
279
top_k=config.num_experts_per_tok,
280
hidden_size=config.hidden_size,
281
intermediate_size=config.intermediate_size,
281
- linear_method=linear_method)
282
+ quant_config=quant_config)
283
self.input_layernorm = RMSNorm(config.hidden_size,
284
eps=config.rms_norm_eps)
285
self.post_attention_layernorm = RMSNorm(config.hidden_size,
@@ -318,7 +319,7 @@ class MixtralModel(nn.Module):
319
def __init__(
320
self,
321
config: MixtralConfig,
321
- linear_method: Optional[LinearMethodBase] = None,
322
+ quant_config: Optional[QuantizationConfig] = None,
323
lora_config: Optional[LoRAConfig] = None,
324
) -> None:
325
super().__init__()
@@ -334,7 +335,7 @@ class MixtralModel(nn.Module):
335
org_num_embeddings=config.vocab_size,
336
)
337
self.layers = nn.ModuleList([
337
- MixtralDecoderLayer(config, linear_method=linear_method)
338
+ MixtralDecoderLayer(config, quant_config=quant_config)
339
for _ in range(config.num_hidden_layers)
340
])
341
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -384,14 +385,13 @@ class MixtralForCausalLM(nn.Module):
385
def __init__(
386
self,
387
config: MixtralConfig,
387
- linear_method: Optional[LinearMethodBase] = None,
388
+ quant_config: Optional[QuantizationConfig] = None,
389
lora_config: Optional[LoRAConfig] = None,
390
) -> None:
391
super().__init__()
392
self.config = config
392
- self.linear_method = linear_method
393
self.model = MixtralModel(config,
394
- linear_method,
394
+ quant_config,
395
lora_config=lora_config)
396
self.unpadded_vocab_size = config.vocab_size
397
if lora_config:
vllm/model_executor/models/mixtral_quant.py
+21
-20
@@ -34,11 +34,12 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size,
35
tensor_model_parallel_all_reduce)
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
-from vllm.model_executor.layers.linear import (LinearMethodBase,
38
- QKVParallelLinear,
37
+from vllm.model_executor.layers.linear import (QKVParallelLinear,
38
ReplicatedLinear,
39
RowParallelLinear)
40
from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
+from vllm.model_executor.layers.quantization.base_config import (
42
+ QuantizationConfig)
43
from vllm.model_executor.layers.rotary_embedding import get_rope
44
from vllm.model_executor.layers.sampler import Sampler
45
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -55,7 +56,7 @@ class MixtralMLP(nn.Module):
56
num_experts: int,
57
hidden_size: int,
58
intermediate_size: int,
58
- linear_method: Optional[LinearMethodBase] = None,
59
+ quant_config: Optional[QuantizationConfig] = None,
60
) -> None:
61
super().__init__()
62
self.num_experts = num_experts
@@ -65,15 +66,15 @@ class MixtralMLP(nn.Module):
66
self.w1 = ReplicatedLinear(self.hidden_dim,
67
self.ffn_dim,
68
bias=False,
68
- linear_method=linear_method)
69
+ quant_config=quant_config)
70
self.w2 = ReplicatedLinear(self.ffn_dim,
71
self.hidden_dim,
72
bias=False,
72
- linear_method=linear_method)
73
+ quant_config=quant_config)
74
self.w3 = ReplicatedLinear(self.hidden_dim,
75
self.ffn_dim,
76
bias=False,
76
- linear_method=linear_method)
77
+ quant_config=quant_config)
78
79
# TODO: Use vllm's SiluAndMul
80
self.act_fn = nn.SiLU()
@@ -92,7 +93,7 @@ class MixtralMoE(nn.Module):
93
def __init__(
94
self,
95
config: MixtralConfig,
95
- linear_method: Optional[LinearMethodBase] = None,
96
+ quant_config: Optional[QuantizationConfig] = None,
97
):
98
super().__init__()
99
self.config = config
@@ -115,14 +116,14 @@ class MixtralMoE(nn.Module):
116
MixtralMLP(self.num_total_experts,
117
config.hidden_size,
118
config.intermediate_size,
118
- linear_method=linear_method)
119
+ quant_config=quant_config)
120
if idx in self.expert_indicies else None
121
for idx in range(self.num_total_experts)
122
])
123
self.gate = ReplicatedLinear(config.hidden_size,
124
self.num_total_experts,
125
bias=False,
125
- linear_method=None)
126
+ quant_config=None)
127
128
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
129
num_tokens, hidden_dim = hidden_states.shape
@@ -162,7 +163,7 @@ class MixtralAttention(nn.Module):
163
num_kv_heads: int,
164
max_position: int = 4096 * 32,
165
rope_theta: float = 10000,
165
- linear_method: Optional[LinearMethodBase] = None,
166
+ quant_config: Optional[QuantizationConfig] = None,
167
sliding_window: Optional[int] = None) -> None:
168
super().__init__()
169
self.hidden_size = hidden_size
@@ -193,13 +194,13 @@ class MixtralAttention(nn.Module):
194
self.total_num_heads,
195
self.total_num_kv_heads,
196
bias=False,
196
- linear_method=linear_method,
197
+ quant_config=quant_config,
198
)
199
self.o_proj = RowParallelLinear(
200
self.total_num_heads * self.head_dim,
201
hidden_size,
202
bias=False,
202
- linear_method=linear_method,
203
+ quant_config=quant_config,
204
)
205
self.rotary_emb = get_rope(
206
self.head_dim,
@@ -236,7 +237,7 @@ class MixtralDecoderLayer(nn.Module):
237
def __init__(
238
self,
239
config: MixtralConfig,
239
- linear_method: Optional[LinearMethodBase] = None,
240
+ quant_config: Optional[QuantizationConfig] = None,
241
) -> None:
242
super().__init__()
243
self.hidden_size = config.hidden_size
@@ -249,9 +250,9 @@ class MixtralDecoderLayer(nn.Module):
250
num_kv_heads=config.num_key_value_heads,
251
rope_theta=rope_theta,
252
sliding_window=config.sliding_window,
252
- linear_method=linear_method)
253
+ quant_config=quant_config)
254
self.block_sparse_moe = MixtralMoE(config=config,
254
- linear_method=linear_method)
255
+ quant_config=quant_config)
256
self.input_layernorm = RMSNorm(config.hidden_size,
257
eps=config.rms_norm_eps)
258
self.post_attention_layernorm = RMSNorm(config.hidden_size,
@@ -291,7 +292,7 @@ class MixtralModel(nn.Module):
292
def __init__(
293
self,
294
config: MixtralConfig,
294
- linear_method: Optional[LinearMethodBase] = None,
295
+ quant_config: Optional[QuantizationConfig] = None,
296
) -> None:
297
super().__init__()
298
self.padding_idx = config.pad_token_id
@@ -302,7 +303,7 @@ class MixtralModel(nn.Module):
303
config.hidden_size,
304
)
305
self.layers = nn.ModuleList([
305
- MixtralDecoderLayer(config, linear_method=linear_method)
306
+ MixtralDecoderLayer(config, quant_config=quant_config)
307
for _ in range(config.num_hidden_layers)
308
])
309
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -331,12 +332,12 @@ class MixtralForCausalLM(nn.Module):
332
def __init__(
333
self,
334
config: MixtralConfig,
334
- linear_method: Optional[LinearMethodBase] = None,
335
+ quant_config: Optional[QuantizationConfig] = None,
336
) -> None:
337
super().__init__()
338
self.config = config
338
- self.linear_method = linear_method
339
- self.model = MixtralModel(config, linear_method)
339
+ self.quant_config = quant_config
340
+ self.model = MixtralModel(config, quant_config)
341
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
342
self.logits_processor = LogitsProcessor(config.vocab_size)
343
self.sampler = Sampler()
vllm/model_executor/models/mpt.py
+17
-16
@@ -11,10 +11,11 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
11
get_tensor_model_parallel_world_size)
12
from vllm.model_executor.layers.activation import get_act_fn
13
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14
- LinearMethodBase,
14
QKVParallelLinear,
15
RowParallelLinear)
16
from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
+from vllm.model_executor.layers.quantization.base_config import (
18
+ QuantizationConfig)
19
from vllm.model_executor.layers.sampler import Sampler
20
from vllm.model_executor.layers.vocab_parallel_embedding import (
21
VocabParallelEmbedding)
@@ -42,7 +43,7 @@ class MPTAttention(nn.Module):
43
def __init__(
44
self,
45
config: MPTConfig,
45
- linear_method: Optional[LinearMethodBase] = None,
46
+ quant_config: Optional[QuantizationConfig] = None,
47
):
48
super().__init__()
49
self.d_model = config.d_model
@@ -65,7 +66,7 @@ class MPTAttention(nn.Module):
66
self.total_num_heads,
67
self.total_num_kv_heads,
68
bias=not config.no_bias,
68
- linear_method=linear_method,
69
+ quant_config=quant_config,
70
)
71
if self.qk_ln:
72
self.q_ln = nn.LayerNorm(self.d_model)
@@ -74,7 +75,7 @@ class MPTAttention(nn.Module):
75
self.d_model,
76
self.d_model,
77
bias=not config.no_bias,
77
- linear_method=linear_method,
78
+ quant_config=quant_config,
79
)
80
81
tp_world_size = get_tensor_model_parallel_world_size()
@@ -133,7 +134,7 @@ class MPTMLP(nn.Module):
134
def __init__(
135
self,
136
config: MPTConfig,
136
- linear_method: Optional[LinearMethodBase] = None,
137
+ quant_config: Optional[QuantizationConfig] = None,
138
):
139
super().__init__()
140
hidden_size = config.d_model
@@ -143,15 +144,15 @@ class MPTMLP(nn.Module):
144
hidden_size,
145
intermediate_size,
146
bias=not config.no_bias,
146
- linear_method=linear_method,
147
+ quant_config=quant_config,
148
)
148
- quant_config = getattr(linear_method, "quant_config", None)
149
+ quant_config = getattr(quant_config, "quant_config", None)
150
self.act = get_act_fn("gelu", quant_config, intermediate_size)
151
self.down_proj = RowParallelLinear(
152
intermediate_size,
153
hidden_size,
154
bias=not config.no_bias,
154
- linear_method=linear_method,
155
+ quant_config=quant_config,
156
)
157
158
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -166,14 +167,14 @@ class MPTBlock(nn.Module):
167
def __init__(
168
self,
169
config: MPTConfig,
169
- linear_method: Optional[LinearMethodBase] = None,
170
+ quant_config: Optional[QuantizationConfig] = None,
171
):
172
super().__init__()
173
hidden_size = config.d_model
174
self.norm_1 = nn.LayerNorm(hidden_size)
174
- self.attn = MPTAttention(config, linear_method)
175
+ self.attn = MPTAttention(config, quant_config)
176
self.norm_2 = nn.LayerNorm(hidden_size)
176
- self.ffn = MPTMLP(config, linear_method)
177
+ self.ffn = MPTMLP(config, quant_config)
178
179
def forward(
180
self,
@@ -201,7 +202,7 @@ class MPTModel(nn.Module):
202
def __init__(
203
self,
204
config: MPTConfig,
204
- linear_method: Optional[LinearMethodBase] = None,
205
+ quant_config: Optional[QuantizationConfig] = None,
206
):
207
super().__init__()
208
assert config.embedding_fraction == 1.0
@@ -212,7 +213,7 @@ class MPTModel(nn.Module):
213
config.d_model,
214
)
215
self.blocks = nn.ModuleList(
215
- [MPTBlock(config, linear_method) for _ in range(config.n_layers)])
216
+ [MPTBlock(config, quant_config) for _ in range(config.n_layers)])
217
self.norm_f = nn.LayerNorm(config.d_model)
218
if config.no_bias:
219
for module in self.modules():
@@ -246,14 +247,14 @@ class MPTForCausalLM(nn.Module):
247
def __init__(
248
self,
249
config: MPTConfig,
249
- linear_method: Optional[LinearMethodBase] = None,
250
+ quant_config: Optional[QuantizationConfig] = None,
251
):
252
super().__init__()
253
self.config = config
254
assert config.tie_word_embeddings
254
- self.linear_method = linear_method
255
+ self.quant_config = quant_config
256
256
- self.transformer = MPTModel(config, linear_method)
257
+ self.transformer = MPTModel(config, quant_config)
258
self.lm_head_weight = self.transformer.wte.weight
259
self.logits_processor = LogitsProcessor(config.vocab_size)
260
self.sampler = Sampler()
vllm/model_executor/models/olmo.py
+16
-16
@@ -30,11 +30,12 @@ from transformers import OlmoConfig
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.distributed import get_tensor_model_parallel_world_size
32
from vllm.model_executor.layers.activation import SiluAndMul
33
-from vllm.model_executor.layers.linear import (LinearMethodBase,
34
- MergedColumnParallelLinear,
33
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
34
QKVParallelLinear,
35
RowParallelLinear)
36
from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
+from vllm.model_executor.layers.quantization.base_config import (
38
+ QuantizationConfig)
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -54,7 +55,7 @@ class OlmoAttention(nn.Module):
55
def __init__(
56
self,
57
config: OlmoConfig,
57
- linear_method: Optional[LinearMethodBase] = None,
58
+ quant_config: Optional[QuantizationConfig] = None,
59
):
60
super().__init__()
61
self.config = config
@@ -79,7 +80,7 @@ class OlmoAttention(nn.Module):
80
self.head_dim,
81
self.total_num_heads,
82
bias=config.attention_bias,
82
- linear_method=linear_method,
83
+ quant_config=quant_config,
84
)
85
86
# Rotary embeddings.
@@ -99,7 +100,7 @@ class OlmoAttention(nn.Module):
100
self.hidden_size,
101
self.hidden_size,
102
bias=config.attention_bias,
102
- linear_method=linear_method,
103
+ quant_config=quant_config,
104
)
105
106
def forward(
@@ -129,7 +130,7 @@ class OlmoMLP(nn.Module):
130
def __init__(
131
self,
132
config: OlmoConfig,
132
- linear_method: Optional[LinearMethodBase] = None,
133
+ quant_config: Optional[QuantizationConfig] = None,
134
):
135
super().__init__()
136
self.config = config
@@ -141,7 +142,7 @@ class OlmoMLP(nn.Module):
142
self.hidden_size,
143
[self.intermediate_size] * 2,
144
bias=False,
144
- linear_method=linear_method,
145
+ quant_config=quant_config,
146
)
147
148
# Activation function.
@@ -152,7 +153,7 @@ class OlmoMLP(nn.Module):
153
self.intermediate_size,
154
self.hidden_size,
155
bias=False,
155
- linear_method=linear_method,
156
+ quant_config=quant_config,
157
)
158
159
def forward(
@@ -174,13 +175,13 @@ class OlmoDecoderLayer(nn.Module):
175
176
def __init__(self,
177
config: OlmoConfig,
177
- linear_method: Optional[LinearMethodBase] = None):
178
+ quant_config: Optional[QuantizationConfig] = None):
179
super().__init__()
180
# Attention block.
180
- self.self_attn = OlmoAttention(config, linear_method)
181
+ self.self_attn = OlmoAttention(config, quant_config)
182
183
# MLP block.
183
- self.mlp = OlmoMLP(config, linear_method)
184
+ self.mlp = OlmoMLP(config, quant_config)
185
186
# LayerNorm
187
self.input_layernorm = nn.LayerNorm(config.hidden_size,
@@ -216,14 +217,14 @@ class OlmoModel(nn.Module):
217
218
def __init__(self,
219
config: OlmoConfig,
219
- linear_method: Optional[LinearMethodBase] = None):
220
+ quant_config: Optional[QuantizationConfig] = None):
221
super().__init__()
222
self.config = config
223
224
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
225
config.hidden_size)
226
self.layers = nn.ModuleList([
226
- OlmoDecoderLayer(config, linear_method)
227
+ OlmoDecoderLayer(config, quant_config)
228
for layer_idx in range(config.num_hidden_layers)
229
])
230
self.norm = nn.LayerNorm(config.hidden_size,
@@ -270,11 +271,10 @@ class OlmoForCausalLM(nn.Module):
271
272
def __init__(self,
273
config: OlmoConfig,
273
- linear_method: Optional[LinearMethodBase] = None):
274
+ quant_config: Optional[QuantizationConfig] = None):
275
super().__init__()
276
self.config = config
276
- self.linear_method = linear_method
277
- self.model = OlmoModel(config, linear_method)
277
+ self.model = OlmoModel(config, quant_config)
278
if config.tie_word_embeddings:
279
self.lm_head_weight = self.model.embed_tokens.weight
280
else:
vllm/model_executor/models/opt.py
+19
-18
@@ -27,11 +27,12 @@ from vllm.attention import Attention, AttentionMetadata
27
from vllm.distributed import get_tensor_model_parallel_world_size
28
from vllm.model_executor.layers.activation import get_act_fn
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
- LinearMethodBase,
30
QKVParallelLinear,
31
ReplicatedLinear,
32
RowParallelLinear)
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
+from vllm.model_executor.layers.quantization.base_config import (
35
+ QuantizationConfig)
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
@@ -60,7 +61,7 @@ class OPTAttention(nn.Module):
61
embed_dim: int,
62
num_heads: int,
63
bias: bool = True,
63
- linear_method: Optional[LinearMethodBase] = None,
64
+ quant_config: Optional[QuantizationConfig] = None,
65
) -> None:
66
super().__init__()
67
self.embed_dim = embed_dim
@@ -77,13 +78,13 @@ class OPTAttention(nn.Module):
78
self.head_dim,
79
total_num_heads,
80
bias=bias,
80
- linear_method=linear_method,
81
+ quant_config=quant_config,
82
)
83
self.out_proj = RowParallelLinear(
84
embed_dim,
85
embed_dim,
86
bias=bias,
86
- linear_method=linear_method,
87
+ quant_config=quant_config,
88
)
89
self.attn = Attention(self.num_heads,
90
self.head_dim,
@@ -107,7 +108,7 @@ class OPTDecoderLayer(nn.Module):
108
def __init__(
109
self,
110
config: OPTConfig,
110
- linear_method: Optional[LinearMethodBase] = None,
111
+ quant_config: Optional[QuantizationConfig] = None,
112
):
113
super().__init__()
114
self.config = config
@@ -116,7 +117,7 @@ class OPTDecoderLayer(nn.Module):
117
embed_dim=self.embed_dim,
118
num_heads=config.num_attention_heads,
119
bias=config.enable_bias,
119
- linear_method=linear_method,
120
+ quant_config=quant_config,
121
)
122
self.do_layer_norm_before = config.do_layer_norm_before
123
@@ -127,16 +128,16 @@ class OPTDecoderLayer(nn.Module):
128
self.embed_dim,
129
config.ffn_dim,
130
bias=config.enable_bias,
130
- linear_method=linear_method,
131
+ quant_config=quant_config,
132
)
132
- quant_config = getattr(linear_method, "quant_config", None)
133
+ quant_config = getattr(quant_config, "quant_config", None)
134
self.activation_fn = get_act_fn(config.activation_function,
135
quant_config, config.ffn_dim)
136
self.fc2 = RowParallelLinear(
137
config.ffn_dim,
138
self.embed_dim,
139
bias=config.enable_bias,
139
- linear_method=linear_method,
140
+ quant_config=quant_config,
141
)
142
self.final_layer_norm = nn.LayerNorm(
143
self.embed_dim,
@@ -181,7 +182,7 @@ class OPTDecoder(nn.Module):
182
def __init__(
183
self,
184
config: OPTConfig,
184
- linear_method: Optional[LinearMethodBase] = None,
185
+ quant_config: Optional[QuantizationConfig] = None,
186
):
187
super().__init__()
188
self.config = config
@@ -202,7 +203,7 @@ class OPTDecoder(nn.Module):
203
self.project_out = ReplicatedLinear(config.hidden_size,
204
config.word_embed_proj_dim,
205
bias=False,
205
- linear_method=linear_method)
206
+ quant_config=quant_config)
207
else:
208
self.project_out = None
209
@@ -210,7 +211,7 @@ class OPTDecoder(nn.Module):
211
self.project_in = ReplicatedLinear(config.word_embed_proj_dim,
212
config.hidden_size,
213
bias=False,
213
- linear_method=linear_method)
214
+ quant_config=quant_config)
215
else:
216
self.project_in = None
217
@@ -226,7 +227,7 @@ class OPTDecoder(nn.Module):
227
self.final_layer_norm = None
228
229
self.layers = nn.ModuleList([
229
- OPTDecoderLayer(config, linear_method)
230
+ OPTDecoderLayer(config, quant_config)
231
for _ in range(config.num_hidden_layers)
232
])
233
@@ -259,10 +260,10 @@ class OPTModel(nn.Module):
260
def __init__(
261
self,
262
config: OPTConfig,
262
- linear_method: Optional[LinearMethodBase] = None,
263
+ quant_config: Optional[QuantizationConfig] = None,
264
):
265
super().__init__()
265
- self.decoder = OPTDecoder(config, linear_method)
266
+ self.decoder = OPTDecoder(config, quant_config)
267
268
def forward(
269
self,
@@ -279,12 +280,12 @@ class OPTForCausalLM(nn.Module):
280
def __init__(
281
self,
282
config,
282
- linear_method: Optional[LinearMethodBase] = None,
283
+ quant_config: Optional[QuantizationConfig] = None,
284
):
285
super().__init__()
286
self.config = config
286
- self.linear_method = linear_method
287
- self.model = OPTModel(config, linear_method)
287
+ self.quant_config = quant_config
288
+ self.model = OPTModel(config, quant_config)
289
self.lm_head_weight = self.model.decoder.embed_tokens.weight
290
self.logits_processor = LogitsProcessor(config.vocab_size)
291
self.sampler = Sampler()
vllm/model_executor/models/orion.py
+17
-16
@@ -13,11 +13,12 @@ from transformers import PretrainedConfig
13
from vllm.attention import Attention, AttentionMetadata
14
from vllm.distributed import get_tensor_model_parallel_world_size
15
from vllm.model_executor.layers.activation import SiluAndMul
16
-from vllm.model_executor.layers.linear import (LinearMethodBase,
17
- MergedColumnParallelLinear,
16
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
17
QKVParallelLinear,
18
RowParallelLinear)
19
from vllm.model_executor.layers.logits_processor import LogitsProcessor
20
+from vllm.model_executor.layers.quantization.base_config import (
21
+ QuantizationConfig)
22
from vllm.model_executor.layers.rotary_embedding import get_rope
23
from vllm.model_executor.layers.sampler import Sampler
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -34,17 +35,17 @@ class OrionMLP(nn.Module):
35
hidden_size: int,
36
intermediate_size: int,
37
hidden_act: str,
37
- linear_method: Optional[LinearMethodBase] = None,
38
+ quant_config: Optional[QuantizationConfig] = None,
39
) -> None:
40
super().__init__()
41
self.gate_up_proj = MergedColumnParallelLinear(
42
hidden_size, [intermediate_size] * 2,
43
bias=False,
43
- linear_method=linear_method)
44
+ quant_config=quant_config)
45
self.down_proj = RowParallelLinear(intermediate_size,
46
hidden_size,
47
bias=False,
47
- linear_method=linear_method)
48
+ quant_config=quant_config)
49
if hidden_act != "silu":
50
raise ValueError(f"Unsupported activation: {hidden_act}. "
51
"Only silu is supported for now.")
@@ -67,7 +68,7 @@ class OrionAttention(nn.Module):
68
rope_theta: float = 10000,
69
rope_scaling: Optional[Dict[str, Any]] = None,
70
max_position_embeddings: int = 8192,
70
- linear_method: Optional[LinearMethodBase] = None,
71
+ quant_config: Optional[QuantizationConfig] = None,
72
) -> None:
73
super().__init__()
74
self.hidden_size = hidden_size
@@ -98,13 +99,13 @@ class OrionAttention(nn.Module):
99
self.total_num_heads,
100
self.total_num_kv_heads,
101
bias=False,
101
- linear_method=linear_method,
102
+ quant_config=quant_config,
103
)
104
self.o_proj = RowParallelLinear(
105
self.total_num_heads * self.head_dim,
106
hidden_size,
107
bias=False,
107
- linear_method=linear_method,
108
+ quant_config=quant_config,
109
)
110
111
self.rotary_emb = get_rope(
@@ -139,7 +140,7 @@ class OrionDecoderLayer(nn.Module):
140
def __init__(
141
self,
142
config: PretrainedConfig,
142
- linear_method: Optional[LinearMethodBase] = None,
143
+ quant_config: Optional[QuantizationConfig] = None,
144
) -> None:
145
super().__init__()
146
self.hidden_size = config.hidden_size
@@ -154,13 +155,13 @@ class OrionDecoderLayer(nn.Module):
155
rope_theta=rope_theta,
156
rope_scaling=rope_scaling,
157
max_position_embeddings=max_position_embeddings,
157
- linear_method=linear_method,
158
+ quant_config=quant_config,
159
)
160
self.mlp = OrionMLP(
161
hidden_size=self.hidden_size,
162
intermediate_size=config.intermediate_size,
163
hidden_act=config.hidden_act,
163
- linear_method=linear_method,
164
+ quant_config=quant_config,
165
)
166
167
self.input_layernorm = nn.LayerNorm(config.hidden_size,
@@ -201,7 +202,7 @@ class OrionModel(nn.Module):
202
def __init__(
203
self,
204
config: PretrainedConfig,
204
- linear_method: Optional[LinearMethodBase] = None,
205
+ quant_config: Optional[QuantizationConfig] = None,
206
) -> None:
207
super().__init__()
208
self.config = config
@@ -212,7 +213,7 @@ class OrionModel(nn.Module):
213
config.hidden_size,
214
)
215
self.layers = nn.ModuleList([
215
- OrionDecoderLayer(config, linear_method)
216
+ OrionDecoderLayer(config, quant_config)
217
for _ in range(config.num_hidden_layers)
218
])
219
self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -244,12 +245,12 @@ class OrionForCausalLM(nn.Module):
245
def __init__(
246
self,
247
config: PretrainedConfig,
247
- linear_method: Optional[LinearMethodBase] = None,
248
+ quant_config: Optional[QuantizationConfig] = None,
249
) -> None:
250
super().__init__()
251
self.config = config
251
- self.linear_method = linear_method
252
- self.model = OrionModel(config, linear_method)
252
+ self.quant_config = quant_config
253
+ self.model = OrionModel(config, quant_config)
254
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
255
self.logits_processor = LogitsProcessor(config.vocab_size)
256
self.sampler = Sampler()
vllm/model_executor/models/phi.py
+18
-17
@@ -45,10 +45,11 @@ from vllm.attention import Attention, AttentionMetadata
45
from vllm.distributed import get_tensor_model_parallel_world_size
46
from vllm.model_executor.layers.activation import get_act_fn
47
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
48
- LinearMethodBase,
48
QKVParallelLinear,
49
RowParallelLinear)
50
from vllm.model_executor.layers.logits_processor import LogitsProcessor
51
+from vllm.model_executor.layers.quantization.base_config import (
52
+ QuantizationConfig)
53
from vllm.model_executor.layers.rotary_embedding import get_rope
54
from vllm.model_executor.layers.sampler import Sampler
55
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -62,7 +63,7 @@ class PhiAttention(nn.Module):
63
64
def __init__(self,
65
config: PretrainedConfig,
65
- linear_method: Optional[LinearMethodBase] = None):
66
+ quant_config: Optional[QuantizationConfig] = None):
67
super().__init__()
68
self.total_num_heads = config.num_attention_heads
69
self.hidden_size = config.hidden_size
@@ -80,12 +81,12 @@ class PhiAttention(nn.Module):
81
self.head_size,
82
self.total_num_heads,
83
bias=True,
83
- linear_method=linear_method,
84
+ quant_config=quant_config,
85
)
86
self.dense = RowParallelLinear(
87
self.hidden_size,
88
self.hidden_size,
88
- linear_method=linear_method,
89
+ quant_config=quant_config,
90
)
91
92
scaling = self.head_size**-0.5
@@ -125,7 +126,7 @@ class PhiMLP(nn.Module):
126
127
def __init__(self,
128
config: PretrainedConfig,
128
- linear_method: Optional[LinearMethodBase] = None):
129
+ quant_config: Optional[QuantizationConfig] = None):
130
super().__init__()
131
132
n_inner = getattr(config, "n_inner", None)
@@ -134,14 +135,14 @@ class PhiMLP(nn.Module):
135
self.fc1 = ColumnParallelLinear(
136
config.hidden_size,
137
n_inner,
137
- linear_method=linear_method,
138
+ quant_config=quant_config,
139
)
140
self.fc2 = RowParallelLinear(
141
n_inner,
142
config.hidden_size,
142
- linear_method=linear_method,
143
+ quant_config=quant_config,
144
)
144
- quant_config = getattr(linear_method, "quant_config", None)
145
+ quant_config = getattr(quant_config, "quant_config", None)
146
self.act = get_act_fn(config.hidden_act, quant_config, n_inner)
147
148
def forward(self, hidden_states):
@@ -155,12 +156,12 @@ class PhiLayer(nn.Module):
156
157
def __init__(self,
158
config: PretrainedConfig,
158
- linear_method: Optional[LinearMethodBase] = None):
159
+ quant_config: Optional[QuantizationConfig] = None):
160
super().__init__()
161
self.input_layernorm = nn.LayerNorm(config.hidden_size,
162
eps=config.layer_norm_eps)
162
- self.self_attn = PhiAttention(config, linear_method)
163
- self.mlp = PhiMLP(config, linear_method)
163
+ self.self_attn = PhiAttention(config, quant_config)
164
+ self.mlp = PhiMLP(config, quant_config)
165
166
def forward(
167
self,
@@ -186,14 +187,14 @@ class PhiModel(nn.Module):
187
188
def __init__(self,
189
config: PretrainedConfig,
189
- linear_method: Optional[LinearMethodBase] = None):
190
+ quant_config: Optional[QuantizationConfig] = None):
191
super().__init__()
192
self.config = config
192
- self.linear_method = linear_method
193
+ self.quant_config = quant_config
194
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
195
config.hidden_size)
196
self.layers = nn.ModuleList([
196
- PhiLayer(config, linear_method)
197
+ PhiLayer(config, quant_config)
198
for _ in range(config.num_hidden_layers)
199
])
200
self.final_layernorm = nn.LayerNorm(config.hidden_size,
@@ -225,12 +226,12 @@ class PhiForCausalLM(nn.Module):
226
227
def __init__(self,
228
config: PretrainedConfig,
228
- linear_method: Optional[LinearMethodBase] = None):
229
+ quant_config: Optional[QuantizationConfig] = None):
230
super().__init__()
231
self.config = config
231
- self.linear_method = linear_method
232
+ self.quant_config = quant_config
233
233
- self.model = PhiModel(config, linear_method)
234
+ self.model = PhiModel(config, quant_config)
235
236
self.lm_head = ParallelLMHead(config.vocab_size,
237
config.hidden_size,
vllm/model_executor/models/qwen.py
+17
-16
@@ -14,11 +14,12 @@ from vllm.attention import Attention, AttentionMetadata
14
from vllm.distributed import get_tensor_model_parallel_world_size
15
from vllm.model_executor.layers.activation import SiluAndMul
16
from vllm.model_executor.layers.layernorm import RMSNorm
17
-from vllm.model_executor.layers.linear import (LinearMethodBase,
18
- MergedColumnParallelLinear,
17
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
18
QKVParallelLinear,
19
RowParallelLinear)
20
from vllm.model_executor.layers.logits_processor import LogitsProcessor
21
+from vllm.model_executor.layers.quantization.base_config import (
22
+ QuantizationConfig)
23
from vllm.model_executor.layers.rotary_embedding import get_rope
24
from vllm.model_executor.layers.sampler import Sampler
25
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -35,17 +36,17 @@ class QWenMLP(nn.Module):
36
hidden_size: int,
37
intermediate_size: int,
38
hidden_act: str = "silu",
38
- linear_method: Optional[LinearMethodBase] = None,
39
+ quant_config: Optional[QuantizationConfig] = None,
40
):
41
super().__init__()
42
self.gate_up_proj = MergedColumnParallelLinear(
43
hidden_size, [intermediate_size] * 2,
44
bias=False,
44
- linear_method=linear_method)
45
+ quant_config=quant_config)
46
self.c_proj = RowParallelLinear(intermediate_size,
47
hidden_size,
48
bias=False,
48
- linear_method=linear_method)
49
+ quant_config=quant_config)
50
if hidden_act != "silu":
51
raise ValueError(f"Unsupported activation: {hidden_act}. "
52
"Only silu is supported for now.")
@@ -67,7 +68,7 @@ class QWenAttention(nn.Module):
68
max_position_embeddings: int,
69
rope_theta: float = 10000,
70
rope_scaling: Optional[Dict[str, Any]] = None,
70
- linear_method: Optional[LinearMethodBase] = None,
71
+ quant_config: Optional[QuantizationConfig] = None,
72
):
73
super().__init__()
74
self.hidden_size = hidden_size
@@ -83,13 +84,13 @@ class QWenAttention(nn.Module):
84
self.head_dim,
85
self.total_num_heads,
86
bias=True,
86
- linear_method=linear_method,
87
+ quant_config=quant_config,
88
)
89
self.c_proj = RowParallelLinear(
90
self.total_num_heads * self.head_dim,
91
hidden_size,
92
bias=False,
92
- linear_method=linear_method,
93
+ quant_config=quant_config,
94
)
95
self.scaling = self.head_dim**-0.5
96
@@ -122,7 +123,7 @@ class QWenBlock(nn.Module):
123
def __init__(
124
self,
125
config: PretrainedConfig,
125
- linear_method: Optional[LinearMethodBase] = None,
126
+ quant_config: Optional[QuantizationConfig] = None,
127
):
128
super().__init__()
129
self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -134,13 +135,13 @@ class QWenBlock(nn.Module):
135
config.max_position_embeddings,
136
rope_theta=rope_theta,
137
rope_scaling=rope_scaling,
137
- linear_method=linear_method)
138
+ quant_config=quant_config)
139
140
self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
141
142
self.mlp = QWenMLP(config.hidden_size,
143
config.intermediate_size // 2,
143
- linear_method=linear_method)
144
+ quant_config=quant_config)
145
146
def forward(
147
self,
@@ -174,7 +175,7 @@ class QWenModel(nn.Module):
175
def __init__(
176
self,
177
config: PretrainedConfig,
177
- linear_method: Optional[LinearMethodBase] = None,
178
+ quant_config: Optional[QuantizationConfig] = None,
179
):
180
super().__init__()
181
self.config = config
@@ -185,7 +186,7 @@ class QWenModel(nn.Module):
186
config.hidden_size,
187
)
188
self.h = nn.ModuleList([
188
- QWenBlock(config, linear_method)
189
+ QWenBlock(config, quant_config)
190
for _ in range(config.num_hidden_layers)
191
])
192
self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -217,12 +218,12 @@ class QWenLMHeadModel(nn.Module):
218
def __init__(
219
self,
220
config: PretrainedConfig,
220
- linear_method: Optional[LinearMethodBase] = None,
221
+ quant_config: Optional[QuantizationConfig] = None,
222
):
223
super().__init__()
224
self.config = config
224
- self.linear_method = linear_method
225
- self.transformer = QWenModel(config, linear_method)
225
+ self.quant_config = quant_config
226
+ self.transformer = QWenModel(config, quant_config)
227
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
228
self.logits_processor = LogitsProcessor(config.vocab_size)
229
self.sampler = Sampler()
vllm/model_executor/models/qwen2.py
+17
-16
@@ -33,11 +33,12 @@ from vllm.config import LoRAConfig
33
from vllm.distributed import get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.layernorm import RMSNorm
36
-from vllm.model_executor.layers.linear import (LinearMethodBase,
37
- MergedColumnParallelLinear,
36
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37
QKVParallelLinear,
38
RowParallelLinear)
39
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
+from vllm.model_executor.layers.quantization.base_config import (
41
+ QuantizationConfig)
42
from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import Sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -54,17 +55,17 @@ class Qwen2MLP(nn.Module):
55
hidden_size: int,
56
intermediate_size: int,
57
hidden_act: str,
57
- linear_method: Optional[LinearMethodBase] = None,
58
+ quant_config: Optional[QuantizationConfig] = None,
59
) -> None:
60
super().__init__()
61
self.gate_up_proj = MergedColumnParallelLinear(
62
hidden_size, [intermediate_size] * 2,
63
bias=False,
63
- linear_method=linear_method)
64
+ quant_config=quant_config)
65
self.down_proj = RowParallelLinear(intermediate_size,
66
hidden_size,
67
bias=False,
67
- linear_method=linear_method)
68
+ quant_config=quant_config)
69
if hidden_act != "silu":
70
raise ValueError(f"Unsupported activation: {hidden_act}. "
71
"Only silu is supported for now.")
@@ -86,7 +87,7 @@ class Qwen2Attention(nn.Module):
87
max_position: int = 4096 * 32,
88
rope_theta: float = 10000,
89
use_sliding_window: bool = False,
89
- linear_method: Optional[LinearMethodBase] = None,
90
+ quant_config: Optional[QuantizationConfig] = None,
91
sliding_window: Optional[int] = None) -> None:
92
super().__init__()
93
self.hidden_size = hidden_size
@@ -117,13 +118,13 @@ class Qwen2Attention(nn.Module):
118
self.total_num_heads,
119
self.total_num_kv_heads,
120
bias=True,
120
- linear_method=linear_method,
121
+ quant_config=quant_config,
122
)
123
self.o_proj = RowParallelLinear(
124
self.total_num_heads * self.head_dim,
125
hidden_size,
126
bias=False,
126
- linear_method=linear_method,
127
+ quant_config=quant_config,
128
)
129
130
self.rotary_emb = get_rope(
@@ -159,7 +160,7 @@ class Qwen2DecoderLayer(nn.Module):
160
self,
161
config: Qwen2Config,
162
layer_idx: int,
162
- linear_method: Optional[LinearMethodBase] = None,
163
+ quant_config: Optional[QuantizationConfig] = None,
164
) -> None:
165
super().__init__()
166
self.hidden_size = config.hidden_size
@@ -174,13 +175,13 @@ class Qwen2DecoderLayer(nn.Module):
175
num_kv_heads=config.num_key_value_heads,
176
rope_theta=rope_theta,
177
use_sliding_window=use_sliding_window,
177
- linear_method=linear_method,
178
+ quant_config=quant_config,
179
sliding_window=config.sliding_window)
180
self.mlp = Qwen2MLP(
181
hidden_size=self.hidden_size,
182
intermediate_size=config.intermediate_size,
183
hidden_act=config.hidden_act,
183
- linear_method=linear_method,
184
+ quant_config=quant_config,
185
)
186
self.input_layernorm = RMSNorm(config.hidden_size,
187
eps=config.rms_norm_eps)
@@ -221,7 +222,7 @@ class Qwen2Model(nn.Module):
222
def __init__(
223
self,
224
config: Qwen2Config,
224
- linear_method: Optional[LinearMethodBase] = None,
225
+ quant_config: Optional[QuantizationConfig] = None,
226
) -> None:
227
super().__init__()
228
self.config = config
@@ -233,7 +234,7 @@ class Qwen2Model(nn.Module):
234
config.hidden_size,
235
)
236
self.layers = nn.ModuleList([
236
- Qwen2DecoderLayer(config, layer_idx, linear_method)
237
+ Qwen2DecoderLayer(config, layer_idx, quant_config)
238
for layer_idx in range(config.num_hidden_layers)
239
])
240
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -286,14 +287,14 @@ class Qwen2ForCausalLM(nn.Module):
287
def __init__(
288
self,
289
config: Qwen2Config,
289
- linear_method: Optional[LinearMethodBase] = None,
290
+ quant_config: Optional[QuantizationConfig] = None,
291
lora_config: Optional[LoRAConfig] = None,
292
) -> None:
293
del lora_config
294
super().__init__()
295
self.config = config
295
- self.linear_method = linear_method
296
- self.model = Qwen2Model(config, linear_method)
296
+ self.quant_config = quant_config
297
+ self.model = Qwen2Model(config, quant_config)
298
299
if config.tie_word_embeddings:
300
self.lm_head_weight = self.model.embed_tokens.weight
vllm/model_executor/models/qwen2_moe.py
+22
-23
@@ -36,12 +36,13 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
36
from vllm.model_executor.layers.activation import SiluAndMul
37
from vllm.model_executor.layers.fused_moe import fused_moe
38
from vllm.model_executor.layers.layernorm import RMSNorm
39
-from vllm.model_executor.layers.linear import (LinearMethodBase,
40
- MergedColumnParallelLinear,
39
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
40
QKVParallelLinear,
41
ReplicatedLinear,
42
RowParallelLinear)
43
from vllm.model_executor.layers.logits_processor import LogitsProcessor
44
+from vllm.model_executor.layers.quantization.base_config import (
45
+ QuantizationConfig)
46
from vllm.model_executor.layers.rotary_embedding import get_rope
47
from vllm.model_executor.layers.sampler import Sampler
48
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -58,18 +59,18 @@ class Qwen2MoeMLP(nn.Module):
59
hidden_size: int,
60
intermediate_size: int,
61
hidden_act: str,
61
- linear_method: Optional[LinearMethodBase] = None,
62
+ quant_config: Optional[QuantizationConfig] = None,
63
reduce_results: bool = True,
64
) -> None:
65
super().__init__()
66
self.gate_up_proj = MergedColumnParallelLinear(
67
hidden_size, [intermediate_size] * 2,
68
bias=False,
68
- linear_method=linear_method)
69
+ quant_config=quant_config)
70
self.down_proj = RowParallelLinear(intermediate_size,
71
hidden_size,
72
bias=False,
72
- linear_method=linear_method,
73
+ quant_config=quant_config,
74
reduce_results=reduce_results)
75
if hidden_act != "silu":
76
raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -88,7 +89,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
89
def __init__(
90
self,
91
config: PretrainedConfig,
91
- linear_method: Optional[LinearMethodBase] = None,
92
+ quant_config: Optional[QuantizationConfig] = None,
93
):
94
super().__init__()
95
self.config = config
@@ -105,7 +106,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
106
Qwen2MoeMLP(hidden_size=config.hidden_size,
107
intermediate_size=config.moe_intermediate_size,
108
hidden_act=config.hidden_act,
108
- linear_method=linear_method,
109
+ quant_config=quant_config,
110
reduce_results=False)
111
for idx in range(self.n_routed_experts)
112
])
@@ -114,13 +115,13 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
115
self.gate = ReplicatedLinear(config.hidden_size,
116
self.n_routed_experts,
117
bias=False,
117
- linear_method=None)
118
+ quant_config=None)
119
if config.shared_expert_intermediate_size > 0:
120
self.shared_expert = Qwen2MoeMLP(
121
hidden_size=config.hidden_size,
122
intermediate_size=config.shared_expert_intermediate_size,
123
hidden_act=config.hidden_act,
123
- linear_method=linear_method,
124
+ quant_config=quant_config,
125
reduce_results=False,
126
)
127
else:
@@ -186,7 +187,7 @@ class Qwen2MoeAttention(nn.Module):
187
rope_theta: float = 10000,
188
rope_scaling: Optional[Dict[str, Any]] = None,
189
max_position_embeddings: int = 8192,
189
- linear_method: Optional[LinearMethodBase] = None,
190
+ quant_config: Optional[QuantizationConfig] = None,
191
) -> None:
192
super().__init__()
193
self.hidden_size = hidden_size
@@ -217,14 +218,14 @@ class Qwen2MoeAttention(nn.Module):
218
self.total_num_heads,
219
self.total_num_kv_heads,
220
bias=True,
220
- linear_method=linear_method,
221
+ quant_config=quant_config,
222
)
223
224
self.o_proj = RowParallelLinear(
225
self.total_num_heads * self.head_dim,
226
hidden_size,
227
bias=False,
227
- linear_method=linear_method,
228
+ quant_config=quant_config,
229
)
230
231
self.rotary_emb = get_rope(
@@ -260,7 +261,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
261
self,
262
config: PretrainedConfig,
263
layer_idx: int,
263
- linear_method: Optional[LinearMethodBase] = None,
264
+ quant_config: Optional[QuantizationConfig] = None,
265
) -> None:
266
super().__init__()
267
self.hidden_size = config.hidden_size
@@ -275,18 +276,18 @@ class Qwen2MoeDecoderLayer(nn.Module):
276
rope_theta=rope_theta,
277
rope_scaling=rope_scaling,
278
max_position_embeddings=max_position_embeddings,
278
- linear_method=linear_method,
279
+ quant_config=quant_config,
280
)
281
if (config.num_experts is not None
282
and (layer_idx + 1) % config.decoder_sparse_step == 0):
283
self.mlp = Qwen2MoeSparseMoeBlock(config=config,
283
- linear_method=linear_method)
284
+ quant_config=quant_config)
285
else:
286
self.mlp = Qwen2MoeMLP(
287
hidden_size=config.hidden_size,
288
intermediate_size=config.intermediate_size,
289
hidden_act=config.hidden_act,
289
- linear_method=linear_method,
290
+ quant_config=quant_config,
291
)
292
self.input_layernorm = RMSNorm(config.hidden_size,
293
eps=config.rms_norm_eps)
@@ -327,7 +328,7 @@ class Qwen2MoeModel(nn.Module):
328
def __init__(
329
self,
330
config: PretrainedConfig,
330
- linear_method: Optional[LinearMethodBase] = None,
331
+ quant_config: Optional[QuantizationConfig] = None,
332
) -> None:
333
super().__init__()
334
self.padding_idx = config.pad_token_id
@@ -338,9 +339,7 @@ class Qwen2MoeModel(nn.Module):
339
config.hidden_size,
340
)
341
self.layers = nn.ModuleList([
341
- Qwen2MoeDecoderLayer(config,
342
- layer_idx,
343
- linear_method=linear_method)
342
+ Qwen2MoeDecoderLayer(config, layer_idx, quant_config=quant_config)
343
for layer_idx in range(config.num_hidden_layers)
344
])
345
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -370,12 +369,12 @@ class Qwen2MoeForCausalLM(nn.Module):
369
def __init__(
370
self,
371
config: PretrainedConfig,
373
- linear_method: Optional[LinearMethodBase] = None,
372
+ quant_config: Optional[QuantizationConfig] = None,
373
) -> None:
374
super().__init__()
375
self.config = config
377
- self.linear_method = linear_method
378
- self.model = Qwen2MoeModel(config, linear_method)
376
+ self.quant_config = quant_config
377
+ self.model = Qwen2MoeModel(config, quant_config)
378
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
379
self.logits_processor = LogitsProcessor(config.vocab_size)
380
self.sampler = Sampler()
vllm/model_executor/models/stablelm.py
+15
-14
@@ -28,11 +28,12 @@ from transformers import PretrainedConfig
28
from vllm.attention import Attention, AttentionMetadata
29
from vllm.distributed import get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import SiluAndMul
31
-from vllm.model_executor.layers.linear import (LinearMethodBase,
32
- MergedColumnParallelLinear,
31
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
32
QKVParallelLinear,
33
RowParallelLinear)
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
+from vllm.model_executor.layers.quantization.base_config import (
36
+ QuantizationConfig)
37
from vllm.model_executor.layers.rotary_embedding import get_rope
38
from vllm.model_executor.layers.sampler import Sampler
39
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -46,7 +47,7 @@ class StablelmMLP(nn.Module):
47
48
def __init__(self,
49
config: PretrainedConfig,
49
- linear_method: Optional[LinearMethodBase] = None) -> None:
50
+ quant_config: Optional[QuantizationConfig] = None) -> None:
51
super().__init__()
52
self.config = config
53
self.hidden_size = config.hidden_size
@@ -54,7 +55,7 @@ class StablelmMLP(nn.Module):
55
self.gate_up_proj = MergedColumnParallelLinear(
56
config.hidden_size, [config.intermediate_size] * 2,
57
bias=False,
57
- linear_method=linear_method)
58
+ quant_config=quant_config)
59
self.down_proj = RowParallelLinear(config.intermediate_size,
60
config.hidden_size,
61
bias=False)
@@ -71,7 +72,7 @@ class StablelmAttention(nn.Module):
72
73
def __init__(self,
74
config: PretrainedConfig,
74
- linear_method: Optional[LinearMethodBase] = None) -> None:
75
+ quant_config: Optional[QuantizationConfig] = None) -> None:
76
super().__init__()
77
self.config = config
78
self.hidden_size = config.hidden_size
@@ -109,11 +110,11 @@ class StablelmAttention(nn.Module):
110
self.total_num_heads,
111
self.total_num_key_value_heads,
112
self.qkv_bias,
112
- linear_method=linear_method)
113
+ quant_config=quant_config)
114
self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
115
self.hidden_size,
116
bias=False,
116
- linear_method=linear_method)
117
+ quant_config=quant_config)
118
self.rotary_emb = get_rope(
119
self.head_dim,
120
rotary_dim=self.rotary_ndims,
@@ -145,11 +146,11 @@ class StablelmDecoderLayer(nn.Module):
146
def __init__(
147
self,
148
config: PretrainedConfig,
148
- linear_method: Optional[LinearMethodBase] = None,
149
+ quant_config: Optional[QuantizationConfig] = None,
150
) -> None:
151
super().__init__()
152
self.self_attn = StablelmAttention(config)
152
- self.mlp = StablelmMLP(config, linear_method)
153
+ self.mlp = StablelmMLP(config, quant_config)
154
norm_eps = getattr(config, "norm_eps",
155
getattr(config, "layer_norm_eps", 1e-05))
156
self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
@@ -187,14 +188,14 @@ class StableLMEpochModel(nn.Module):
188
189
def __init__(self,
190
config: PretrainedConfig,
190
- linear_method: Optional[LinearMethodBase] = None) -> None:
191
+ quant_config: Optional[QuantizationConfig] = None) -> None:
192
super().__init__()
193
self.embed_tokens = VocabParallelEmbedding(
194
config.vocab_size,
195
config.hidden_size,
196
)
197
self.layers = nn.ModuleList([
197
- StablelmDecoderLayer(config, linear_method)
198
+ StablelmDecoderLayer(config, quant_config)
199
for _ in range(config.num_hidden_layers)
200
])
201
norm_eps = getattr(config, "norm_eps",
@@ -226,12 +227,12 @@ class StablelmForCausalLM(nn.Module):
227
def __init__(
228
self,
229
config: PretrainedConfig,
229
- linear_method: Optional[LinearMethodBase] = None,
230
+ quant_config: Optional[QuantizationConfig] = None,
231
) -> None:
232
super().__init__()
233
self.config = config
233
- self.linear_method = linear_method
234
- self.model = StableLMEpochModel(config, linear_method)
234
+ self.quant_config = quant_config
235
+ self.model = StableLMEpochModel(config, quant_config)
236
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
237
self.logits_processor = LogitsProcessor(config.vocab_size)
238
self.sampler = Sampler()
vllm/model_executor/models/starcoder2.py
+16
-16
@@ -28,10 +28,11 @@ from vllm.attention import Attention, AttentionMetadata
28
from vllm.distributed import get_tensor_model_parallel_world_size
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
- LinearMethodBase,
31
QKVParallelLinear,
32
RowParallelLinear)
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
+from vllm.model_executor.layers.quantization.base_config import (
35
+ QuantizationConfig)
36
from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -45,7 +46,7 @@ class Starcoder2Attention(nn.Module):
46
47
def __init__(self,
48
config: Starcoder2Config,
48
- linear_method: Optional[LinearMethodBase] = None):
49
+ quant_config: Optional[QuantizationConfig] = None):
50
super().__init__()
51
self.config = config
52
@@ -79,13 +80,13 @@ class Starcoder2Attention(nn.Module):
80
self.total_num_heads,
81
self.total_num_kv_heads,
82
bias=self.use_bias,
82
- linear_method=linear_method,
83
+ quant_config=quant_config,
84
)
85
self.o_proj = RowParallelLinear(
86
self.total_num_heads * self.head_dim,
87
self.hidden_size,
88
bias=self.use_bias,
88
- linear_method=linear_method,
89
+ quant_config=quant_config,
90
)
91
self.rotary_emb = get_rope(
92
self.head_dim,
@@ -121,21 +122,21 @@ class Starcoder2MLP(nn.Module):
122
123
def __init__(self,
124
config: Starcoder2Config,
124
- linear_method: Optional[LinearMethodBase] = None):
125
+ quant_config: Optional[QuantizationConfig] = None):
126
super().__init__()
127
self.c_fc = ColumnParallelLinear(
128
config.hidden_size,
129
config.intermediate_size,
130
bias=config.use_bias,
130
- linear_method=linear_method,
131
+ quant_config=quant_config,
132
)
133
self.c_proj = RowParallelLinear(
134
config.intermediate_size,
135
config.hidden_size,
136
bias=config.use_bias,
136
- linear_method=linear_method,
137
+ quant_config=quant_config,
138
)
138
- quant_config = getattr(linear_method, "quant_config", None)
139
+ quant_config = getattr(quant_config, "quant_config", None)
140
self.act = get_act_fn(config.hidden_act, quant_config,
141
config.intermediate_size)
142
@@ -150,12 +151,11 @@ class Starcoder2DecoderLayer(nn.Module):
151
152
def __init__(self,
153
config: Starcoder2Config,
153
- linear_method: Optional[LinearMethodBase] = None):
154
+ quant_config: Optional[QuantizationConfig] = None):
155
super().__init__()
156
self.hidden_size = config.hidden_size
156
- self.self_attn = Starcoder2Attention(config,
157
- linear_method=linear_method)
158
- self.mlp = Starcoder2MLP(config, linear_method=linear_method)
157
+ self.self_attn = Starcoder2Attention(config, quant_config=quant_config)
158
+ self.mlp = Starcoder2MLP(config, quant_config=quant_config)
159
self.input_layernorm = nn.LayerNorm(config.hidden_size,
160
eps=config.norm_epsilon)
161
self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
@@ -192,7 +192,7 @@ class Starcoder2Model(nn.Module):
192
193
def __init__(self,
194
config: Starcoder2Config,
195
- linear_method: Optional[LinearMethodBase] = None):
195
+ quant_config: Optional[QuantizationConfig] = None):
196
super().__init__()
197
self.config = config
198
self.padding_idx = config.pad_token_id
@@ -202,7 +202,7 @@ class Starcoder2Model(nn.Module):
202
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
203
config.hidden_size)
204
self.layers = nn.ModuleList([
205
- Starcoder2DecoderLayer(config, linear_method=linear_method)
205
+ Starcoder2DecoderLayer(config, quant_config=quant_config)
206
for _ in range(config.num_hidden_layers)
207
])
208
self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon)
@@ -227,10 +227,10 @@ class Starcoder2ForCausalLM(nn.Module):
227
228
def __init__(self,
229
config: Starcoder2Config,
230
- linear_method: Optional[LinearMethodBase] = None):
230
+ quant_config: Optional[QuantizationConfig] = None):
231
super().__init__()
232
self.config = config
233
- self.model = Starcoder2Model(config, linear_method=linear_method)
233
+ self.model = Starcoder2Model(config, quant_config=quant_config)
234
self.vocab_size = config.vocab_size
235
self.unpadded_vocab_size = config.vocab_size
236
if config.tie_word_embeddings:
vllm/model_executor/models/xverse.py
+17
-16
@@ -31,11 +31,12 @@ from vllm.config import LoRAConfig
31
from vllm.distributed import get_tensor_model_parallel_world_size
32
from vllm.model_executor.layers.activation import SiluAndMul
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
-from vllm.model_executor.layers.linear import (LinearMethodBase,
35
- MergedColumnParallelLinear,
34
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
35
QKVParallelLinear,
36
RowParallelLinear)
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
+from vllm.model_executor.layers.quantization.base_config import (
39
+ QuantizationConfig)
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -52,17 +53,17 @@ class XverseMLP(nn.Module):
53
hidden_size: int,
54
intermediate_size: int,
55
hidden_act: str,
55
- linear_method: Optional[LinearMethodBase] = None,
56
+ quant_config: Optional[QuantizationConfig] = None,
57
) -> None:
58
super().__init__()
59
self.gate_up_proj = MergedColumnParallelLinear(
60
hidden_size, [intermediate_size] * 2,
61
bias=False,
61
- linear_method=linear_method)
62
+ quant_config=quant_config)
63
self.down_proj = RowParallelLinear(intermediate_size,
64
hidden_size,
65
bias=False,
65
- linear_method=linear_method)
66
+ quant_config=quant_config)
67
if hidden_act != "silu":
68
raise ValueError(f"Unsupported activation: {hidden_act}. "
69
"Only silu is supported for now.")
@@ -85,7 +86,7 @@ class XverseAttention(nn.Module):
86
rope_theta: float = 10000,
87
rope_scaling: Optional[Dict[str, Any]] = None,
88
max_position_embeddings: int = 8192,
88
- linear_method: Optional[LinearMethodBase] = None,
89
+ quant_config: Optional[QuantizationConfig] = None,
90
bias: bool = False,
91
sliding_window: Optional[int] = None,
92
) -> None:
@@ -112,13 +113,13 @@ class XverseAttention(nn.Module):
113
self.total_num_heads,
114
self.total_num_kv_heads,
115
bias=bias,
115
- linear_method=linear_method,
116
+ quant_config=quant_config,
117
)
118
self.o_proj = RowParallelLinear(
119
self.total_num_heads * self.head_dim,
120
hidden_size,
121
bias=bias,
121
- linear_method=linear_method,
122
+ quant_config=quant_config,
123
)
124
125
self.rotary_emb = get_rope(
@@ -154,7 +155,7 @@ class XverseDecoderLayer(nn.Module):
155
def __init__(
156
self,
157
config: PretrainedConfig,
157
- linear_method: Optional[LinearMethodBase] = None,
158
+ quant_config: Optional[QuantizationConfig] = None,
159
) -> None:
160
super().__init__()
161
self.hidden_size = config.hidden_size
@@ -171,7 +172,7 @@ class XverseDecoderLayer(nn.Module):
172
rope_theta=rope_theta,
173
rope_scaling=rope_scaling,
174
max_position_embeddings=max_position_embeddings,
174
- linear_method=linear_method,
175
+ quant_config=quant_config,
176
bias=getattr(config, "bias", False),
177
sliding_window=sliding_window,
178
)
@@ -179,7 +180,7 @@ class XverseDecoderLayer(nn.Module):
180
hidden_size=self.hidden_size,
181
intermediate_size=config.intermediate_size,
182
hidden_act=config.hidden_act,
182
- linear_method=linear_method,
183
+ quant_config=quant_config,
184
)
185
self.input_layernorm = RMSNorm(config.hidden_size,
186
eps=config.rms_norm_eps)
@@ -220,7 +221,7 @@ class XverseModel(nn.Module):
221
def __init__(
222
self,
223
config: PretrainedConfig,
223
- linear_method: Optional[LinearMethodBase] = None,
224
+ quant_config: Optional[QuantizationConfig] = None,
225
lora_config: Optional[LoRAConfig] = None,
226
) -> None:
227
super().__init__()
@@ -236,7 +237,7 @@ class XverseModel(nn.Module):
237
org_num_embeddings=config.vocab_size,
238
)
239
self.layers = nn.ModuleList([
239
- XverseDecoderLayer(config, linear_method)
240
+ XverseDecoderLayer(config, quant_config)
241
for _ in range(config.num_hidden_layers)
242
])
243
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -294,13 +295,13 @@ class XverseForCausalLM(nn.Module):
295
def __init__(
296
self,
297
config: PretrainedConfig,
297
- linear_method: Optional[LinearMethodBase] = None,
298
+ quant_config: Optional[QuantizationConfig] = None,
299
lora_config=None,
300
) -> None:
301
super().__init__()
302
self.config = config
302
- self.linear_method = linear_method
303
- self.model = XverseModel(config, linear_method)
303
+ self.quant_config = quant_config
304
+ self.model = XverseModel(config, quant_config)
305
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
306
self.logits_processor = LogitsProcessor(config.vocab_size)
307
self.sampler = Sampler()