A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] Set `linear_weights` directly on the layer (#3977)
Antoni Baum committed
Apr 11, 2024 at 13:35 UTC
a10d3056da644c31e4ebf95a2b6ad65a626a7350
8 files changed
+114
-102
csrc/quantization/gptq/q_gemm.cu
+1
-1
@@ -2067,7 +2067,7 @@ void gptq_shuffle
2067
const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
2068
vllm::gptq::shuffle_exllama_weight(
2069
(uint32_t*) q_weight.data_ptr(),
2070
- q_perm.device().is_meta() ? NULL : (int*) q_perm.data_ptr(),
2070
+ q_perm.device().is_meta() || q_perm.numel() == 0 ? NULL : (int*) q_perm.data_ptr(),
2071
q_weight.size(0) * 32 / bit,
2072
q_weight.size(1),
2073
bit
tests/kernels/test_moe.py
+1
-1
@@ -73,7 +73,7 @@ def test_mixtral_moe(dtype: torch.dtype):
73
).cuda()
74
75
# Load the weights
76
- vllm_moe.gate.linear_weights["weight"][:] = hf_moe.gate.weight.data
76
+ vllm_moe.gate.weight.data[:] = hf_moe.gate.weight.data
77
for i in range(config.num_local_experts):
78
weights = (hf_moe.experts[i].w1.weight.data,
79
hf_moe.experts[i].w3.weight.data)
vllm/lora/layers.py
+4
-8
@@ -368,7 +368,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
368
def apply_weights(self, x: torch.Tensor,
369
bias: Optional[torch.Tensor]) -> torch.Tensor:
370
output = self.base_layer.linear_method.apply_weights(
371
- self.base_layer.linear_weights, x, bias)
371
+ self.base_layer, x, bias)
372
_apply_lora(
373
x,
374
self.lora_a_stacked,
@@ -402,10 +402,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
402
if self.base_layer.skip_bias_add else None)
403
return output, output_bias
404
405
- @property
406
- def linear_weights(self):
407
- return self.base_layer.linear_weights
408
-
405
@classmethod
406
def can_replace_layer(cls, source_layer: nn.Module,
407
lora_config: LoRAConfig, packed_modules_list: List,
@@ -505,7 +501,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
501
def apply_weights(self, x: torch.Tensor,
502
bias: Optional[torch.Tensor]) -> torch.Tensor:
503
output = self.base_layer.linear_method.apply_weights(
508
- self.base_layer.linear_weights, x, bias)
504
+ self.base_layer, x, bias)
505
_apply_lora_packed_nslice(
506
x,
507
self.lora_a_stacked,
@@ -746,7 +742,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
742
def apply_weights(self, x: torch.Tensor,
743
bias: Optional[torch.Tensor]) -> torch.Tensor:
744
output = self.base_layer.linear_method.apply_weights(
749
- self.base_layer.linear_weights, x, bias)
745
+ self.base_layer, x, bias)
746
_apply_lora_packed_nslice(
747
x,
748
self.lora_a_stacked,
@@ -838,7 +834,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
834
835
def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
836
output = self.base_layer.linear_method.apply_weights(
841
- self.base_layer.linear_weights, x)
837
+ self.base_layer, x)
838
_apply_lora(
839
x,
840
self.lora_a_stacked,
vllm/model_executor/layers/linear.py
+40
-37
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Any, Dict, List, Optional
2
+from typing import List, Optional
3
4
import torch
5
import torch.nn.functional as F
@@ -28,19 +28,24 @@ class LinearMethodBase(ABC):
28
"""Base class for different (maybe quantized) linear methods."""
29
30
@abstractmethod
31
- def create_weights(self, input_size_per_partition: int,
31
+ def create_weights(self, layer: torch.nn.Module,
32
+ input_size_per_partition: int,
33
output_size_per_partition: int, input_size: int,
33
- output_size: int,
34
- params_dtype: torch.dtype) -> Dict[str, Any]:
35
- """Create weights for a linear layer."""
34
+ output_size: int, params_dtype: torch.dtype,
35
+ **extra_weight_attrs):
36
+ """Create weights for a linear layer.
37
+
38
+ The weights will be set as attributes of the layer."""
39
raise NotImplementedError
40
41
@abstractmethod
42
def apply_weights(self,
40
- weights: Dict[str, torch.Tensor],
43
+ layer: torch.nn.Module,
44
x: torch.Tensor,
45
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
43
- """Apply the weights to the input tensor."""
46
+ """Apply the weights in layer to the input tensor.
47
+
48
+ Expects create_weights to have been called before on the layer."""
49
raise NotImplementedError
50
51
@@ -55,22 +60,24 @@ class UnquantizedLinearMethod(LinearMethodBase):
60
def __init__(self, separate_bias_add: bool = False):
61
self.separate_bias_add = separate_bias_add
62
58
- def create_weights(self, input_size_per_partition: int,
63
+ def create_weights(self, layer: torch.nn.Module,
64
+ input_size_per_partition: int,
65
output_size_per_partition: int, input_size: int,
60
- output_size: int,
61
- params_dtype: torch.dtype) -> Dict[str, Any]:
66
+ output_size: int, params_dtype: torch.dtype,
67
+ **extra_weight_attrs):
68
weight = Parameter(torch.empty(output_size_per_partition,
69
input_size_per_partition,
70
dtype=params_dtype),
71
requires_grad=False)
72
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
67
- return {"weight": weight}
73
+ layer.register_parameter("weight", weight)
74
+ set_weight_attrs(weight, extra_weight_attrs)
75
76
def apply_weights(self,
70
- weights: Dict[str, torch.Tensor],
77
+ layer: torch.nn.Module,
78
x: torch.Tensor,
79
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
73
- weight = weights["weight"]
80
+ weight = layer.weight
81
if self.separate_bias_add:
82
if bias is not None:
83
return F.linear(x, weight) + bias
@@ -111,12 +118,9 @@ class ReplicatedLinear(torch.nn.Module):
118
if linear_method is None:
119
linear_method = UnquantizedLinearMethod()
120
self.linear_method = linear_method
114
- self.linear_weights = self.linear_method.create_weights(
115
- self.input_size, self.output_size, self.input_size,
116
- self.output_size, self.params_dtype)
117
- for name, weight in self.linear_weights.items():
118
- if isinstance(weight, torch.Tensor):
119
- self.register_parameter(name, weight)
121
+ self.linear_method.create_weights(self, self.input_size,
122
+ self.output_size, self.input_size,
123
+ self.output_size, self.params_dtype)
124
if bias:
125
self.bias = Parameter(
126
torch.empty(self.output_size, dtype=self.params_dtype))
@@ -126,7 +130,7 @@ class ReplicatedLinear(torch.nn.Module):
130
131
def forward(self, x: torch.Tensor) -> torch.Tensor:
132
bias = self.bias if not self.skip_bias_add else None
129
- output = self.linear_method.apply_weights(self.linear_weights, x, bias)
133
+ output = self.linear_method.apply_weights(self, x, bias)
134
output_bias = self.bias if self.skip_bias_add else None
135
return output, output_bias
136
@@ -177,13 +181,13 @@ class ColumnParallelLinear(torch.nn.Module):
181
if linear_method is None:
182
linear_method = UnquantizedLinearMethod()
183
self.linear_method = linear_method
180
- self.linear_weights = self.linear_method.create_weights(
181
- self.input_size, self.output_size_per_partition, self.input_size,
182
- self.output_size, self.params_dtype)
183
- for name, weight in self.linear_weights.items():
184
- if isinstance(weight, torch.Tensor):
185
- self.register_parameter(name, weight)
186
- set_weight_attrs(weight, {"weight_loader": self.weight_loader})
184
+ self.linear_method.create_weights(self,
185
+ self.input_size,
186
+ self.output_size_per_partition,
187
+ self.input_size,
188
+ self.output_size,
189
+ self.params_dtype,
190
+ weight_loader=self.weight_loader)
191
if bias:
192
self.bias = Parameter(
193
torch.empty(self.output_size_per_partition,
@@ -211,8 +215,7 @@ class ColumnParallelLinear(torch.nn.Module):
215
bias = self.bias if not self.skip_bias_add else None
216
217
# Matrix multiply.
214
- output_parallel = self.linear_method.apply_weights(
215
- self.linear_weights, input_, bias)
218
+ output_parallel = self.linear_method.apply_weights(self, input_, bias)
219
if self.gather_output:
220
# All-gather across the partitions.
221
output = tensor_model_parallel_all_gather(output_parallel)
@@ -523,13 +526,13 @@ class RowParallelLinear(torch.nn.Module):
526
if linear_method is None:
527
linear_method = UnquantizedLinearMethod()
528
self.linear_method = linear_method
526
- self.linear_weights = self.linear_method.create_weights(
527
- self.input_size_per_partition, self.output_size, self.input_size,
528
- self.output_size, self.params_dtype)
529
- for name, weight in self.linear_weights.items():
530
- if isinstance(weight, torch.Tensor):
531
- self.register_parameter(name, weight)
532
- set_weight_attrs(weight, {"weight_loader": self.weight_loader})
529
+ self.linear_method.create_weights(self,
530
+ self.input_size_per_partition,
531
+ self.output_size,
532
+ self.input_size,
533
+ self.output_size,
534
+ self.params_dtype,
535
+ weight_loader=self.weight_loader)
536
537
if not reduce_results and (bias and not skip_bias_add):
538
raise ValueError("When not reduce the results, adding bias to the "
@@ -569,7 +572,7 @@ class RowParallelLinear(torch.nn.Module):
572
573
# Matrix multiply.
574
output_parallel = self.linear_method.apply_weights(
572
- self.linear_weights, input_parallel)
575
+ self, input_parallel)
576
if self.reduce_results and self.tp_size > 1:
577
output_ = tensor_model_parallel_all_reduce(output_parallel)
578
else:
vllm/model_executor/layers/quantization/awq.py
+16
-13
@@ -79,10 +79,11 @@ class AWQLinearMethod(LinearMethodBase):
79
def __init__(self, quant_config: AWQConfig):
80
self.quant_config = quant_config
81
82
- def create_weights(self, input_size_per_partition: int,
82
+ def create_weights(self, layer: torch.nn.Module,
83
+ input_size_per_partition: int,
84
output_size_per_partition: int, input_size: int,
84
- output_size: int,
85
- params_dtype: torch.dtype) -> Dict[str, Any]:
85
+ output_size: int, params_dtype: torch.dtype,
86
+ **extra_weight_attrs):
87
if input_size_per_partition % self.quant_config.group_size != 0:
88
raise ValueError(
89
"The input size is not aligned with the quantized "
@@ -136,19 +137,21 @@ class AWQLinearMethod(LinearMethodBase):
137
"input_dim": 0,
138
"output_dim": 1,
139
})
139
- return {
140
- "qweight": qweight,
141
- "qzeros": qzeros,
142
- "scales": scales,
143
- }
140
+
141
+ layer.register_parameter("qweight", qweight)
142
+ set_weight_attrs(qweight, extra_weight_attrs)
143
+ layer.register_parameter("qzeros", qzeros)
144
+ set_weight_attrs(qzeros, extra_weight_attrs)
145
+ layer.register_parameter("scales", scales)
146
+ set_weight_attrs(scales, extra_weight_attrs)
147
148
def apply_weights(self,
146
- weights: Dict[str, Any],
149
+ layer: torch.nn.Module,
150
x: torch.Tensor,
151
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
149
- qweight = weights["qweight"]
150
- scales = weights["scales"]
151
- qzeros = weights["qzeros"]
152
+ qweight = layer.qweight
153
+ scales = layer.scales
154
+ qzeros = layer.qzeros
155
pack_factor = self.quant_config.pack_factor
156
out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, ))
157
reshaped_x = x.reshape(-1, x.shape[-1])
@@ -163,5 +166,5 @@ class AWQLinearMethod(LinearMethodBase):
166
out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros,
167
pack_factor)
168
if bias is not None:
166
- out = out + bias
169
+ out.add_(bias)
170
return out.reshape(out_shape)
vllm/model_executor/layers/quantization/gptq.py
+26
-21
@@ -89,12 +89,14 @@ class GPTQLinearMethod(LinearMethodBase):
89
90
def create_weights(
91
self,
92
+ layer: torch.nn.Module,
93
input_size_per_partition: int,
94
output_size_per_partition: int,
95
input_size: int,
96
output_size: int,
97
params_dtype: torch.dtype,
97
- ) -> Dict[str, Any]:
98
+ **extra_weight_attrs,
99
+ ):
100
del output_size # Unused.
101
if input_size_per_partition % self.quant_config.group_size != 0:
102
raise ValueError(
@@ -179,37 +181,40 @@ class GPTQLinearMethod(LinearMethodBase):
181
"input_dim": scale_and_zero_input_dim,
182
"output_dim": 1,
183
})
182
- return {
183
- "qweight": qweight,
184
- "g_idx": g_idx,
185
- "qzeros": qzeros,
186
- "scales": scales,
187
- "exllama_state": exllama_state,
188
- }
184
+
185
+ layer.register_parameter("qweight", qweight)
186
+ set_weight_attrs(qweight, extra_weight_attrs)
187
+ layer.register_parameter("g_idx", g_idx)
188
+ set_weight_attrs(g_idx, extra_weight_attrs)
189
+ layer.register_parameter("qzeros", qzeros)
190
+ set_weight_attrs(qzeros, extra_weight_attrs)
191
+ layer.register_parameter("scales", scales)
192
+ set_weight_attrs(scales, extra_weight_attrs)
193
+
194
+ layer.exllama_state = exllama_state
195
196
def apply_weights(self,
191
- weights: Dict[str, Any],
197
+ layer: torch.nn.Module,
198
x: torch.Tensor,
199
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
194
- qweight = weights["qweight"]
200
+ qweight = layer.qweight
201
out_shape = x.shape[:-1] + (qweight.shape[-1], )
202
reshaped_x = x.reshape(-1, x.shape[-1])
203
# exllama needs to shuffle the weight after the weight is loaded
204
# here we do the shuffle on first forward pass
199
- if weights["exllama_state"] == ExllamaState.UNINITIALIZED:
205
+ if layer.exllama_state == ExllamaState.UNINITIALIZED:
206
if self.quant_config.desc_act:
201
- weights["g_idx"] = torch.argsort(weights["g_idx"]).to(
202
- torch.int)
207
+ layer.g_idx.data = torch.argsort(layer.g_idx).to(torch.int)
208
else:
204
- weights["g_idx"] = torch.empty((1, 1), device="meta")
205
- weights["exllama_state"] = ExllamaState.READY
206
- ops.gptq_shuffle(weights["qweight"], weights["g_idx"],
209
+ layer.g_idx.data = torch.empty((0, ),
210
+ device=layer.g_idx.device)
211
+ layer.exllama_state = ExllamaState.READY
212
+ ops.gptq_shuffle(layer.qweight, layer.g_idx,
213
self.quant_config.weight_bits)
208
- output = ops.gptq_gemm(reshaped_x, weights["qweight"],
209
- weights["qzeros"], weights["scales"],
210
- weights["g_idx"],
211
- weights["exllama_state"] == ExllamaState.READY,
214
+ output = ops.gptq_gemm(reshaped_x, layer.qweight, layer.qzeros,
215
+ layer.scales, layer.g_idx,
216
+ layer.exllama_state == ExllamaState.READY,
217
self.quant_config.weight_bits)
218
if bias is not None:
214
- output = output + bias
219
+ output.add_(bias)
220
return output.reshape(out_shape)
vllm/model_executor/layers/quantization/marlin.py
+13
-10
@@ -91,12 +91,14 @@ class MarlinLinearMethod(LinearMethodBase):
91
92
def create_weights(
93
self,
94
+ layer: torch.nn.Module,
95
input_size_per_partition: int,
96
output_size_per_partition: int,
97
input_size: int,
98
output_size: int,
99
params_dtype: torch.dtype,
99
- ) -> Dict[str, Any]:
100
+ **extra_weight_attrs,
101
+ ):
102
del output_size # Unused.
103
104
if params_dtype != torch.float16:
@@ -187,21 +189,22 @@ class MarlinLinearMethod(LinearMethodBase):
189
dtype=torch.int),
190
requires_grad=False)
191
190
- return {
191
- "B": qweight,
192
- "s": scales,
193
- "workspace": workspace,
194
- }
192
+ layer.register_parameter("B", qweight)
193
+ set_weight_attrs(qweight, extra_weight_attrs)
194
+ layer.register_parameter("s", scales)
195
+ set_weight_attrs(scales, extra_weight_attrs)
196
+ layer.register_parameter("workspace", workspace)
197
+ set_weight_attrs(workspace, extra_weight_attrs)
198
199
def apply_weights(
200
self,
198
- weights: Dict[str, Any],
201
+ layer: torch.nn.Module,
202
x: torch.Tensor,
203
bias: Optional[torch.Tensor] = None,
204
) -> torch.Tensor:
202
- qweight = weights["B"]
203
- scales = weights["s"]
204
- workspace = weights["workspace"]
205
+ qweight = layer.B
206
+ scales = layer.s
207
+ workspace = layer.workspace
208
209
x_2d = x.view(-1, x.shape[-1])
210
vllm/model_executor/layers/quantization/squeezellm.py
+13
-11
@@ -68,10 +68,11 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
68
def __init__(self, quant_config: SqueezeLLMConfig):
69
self.quant_config = quant_config
70
71
- def create_weights(self, input_size_per_partition: int,
71
+ def create_weights(self, layer: torch.nn.Module,
72
+ input_size_per_partition: int,
73
output_size_per_partition: int, input_size: int,
73
- output_size: int,
74
- params_dtype: torch.dtype) -> Dict[str, Any]:
74
+ output_size: int, params_dtype: torch.dtype,
75
+ **extra_weight_attrs):
76
if input_size_per_partition % self.quant_config.pack_factor != 0:
77
raise ValueError(
78
"The input size is not aligned with the quantized "
@@ -103,17 +104,18 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
104
set_weight_attrs(lookup_table, {
105
"output_dim": 0,
106
})
106
- return {
107
- "qweight": qweight,
108
- "lookup_table": lookup_table,
109
- }
107
+
108
+ layer.register_parameter("qweight", qweight)
109
+ set_weight_attrs(qweight, extra_weight_attrs)
110
+ layer.register_parameter("lookup_table", lookup_table)
111
+ set_weight_attrs(lookup_table, extra_weight_attrs)
112
113
def apply_weights(self,
112
- weights: Dict[str, Any],
114
+ layer: torch.nn.Module,
115
x: torch.Tensor,
116
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
115
- qweight = weights["qweight"]
116
- lookup_table = weights["lookup_table"]
117
+ qweight = layer.qweight
118
+ lookup_table = layer.lookup_table
119
out_shape = x.shape[:-1] + (qweight.shape[-1], )
120
reshaped_x = x.reshape(-1, x.shape[-1])
121
if is_hip():
@@ -126,5 +128,5 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
128
ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
129
130
if bias is not None:
129
- out = out + bias
131
+ out.add_(bias)
132
return out.reshape(out_shape)