@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a10d3056da
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] Set `linear_weights` directly on the layer (#3977)

Antoni Baum committed Apr 11, 2024 at 13:35 UTC a10d3056da644c31e4ebf95a2b6ad65a626a7350
8 files changed +114 -102
csrc/quantization/gptq/q_gemm.cu
+1 -1
@@ -2067,7 +2067,7 @@ void gptq_shuffle
2067 const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight));
2068 vllm::gptq::shuffle_exllama_weight(
2069 (uint32_t*) q_weight.data_ptr(),
2070 - q_perm.device().is_meta() ? NULL : (int*) q_perm.data_ptr(),
2070 + q_perm.device().is_meta() || q_perm.numel() == 0 ? NULL : (int*) q_perm.data_ptr(),
2071 q_weight.size(0) * 32 / bit,
2072 q_weight.size(1),
2073 bit
tests/kernels/test_moe.py
+1 -1
@@ -73,7 +73,7 @@ def test_mixtral_moe(dtype: torch.dtype):
73 ).cuda()
74
75 # Load the weights
76 - vllm_moe.gate.linear_weights["weight"][:] = hf_moe.gate.weight.data
76 + vllm_moe.gate.weight.data[:] = hf_moe.gate.weight.data
77 for i in range(config.num_local_experts):
78 weights = (hf_moe.experts[i].w1.weight.data,
79 hf_moe.experts[i].w3.weight.data)
vllm/lora/layers.py
+4 -8
@@ -368,7 +368,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
368 def apply_weights(self, x: torch.Tensor,
369 bias: Optional[torch.Tensor]) -> torch.Tensor:
370 output = self.base_layer.linear_method.apply_weights(
371 - self.base_layer.linear_weights, x, bias)
371 + self.base_layer, x, bias)
372 _apply_lora(
373 x,
374 self.lora_a_stacked,
@@ -402,10 +402,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
402 if self.base_layer.skip_bias_add else None)
403 return output, output_bias
404
405 - @property
406 - def linear_weights(self):
407 - return self.base_layer.linear_weights
408 -
405 @classmethod
406 def can_replace_layer(cls, source_layer: nn.Module,
407 lora_config: LoRAConfig, packed_modules_list: List,
@@ -505,7 +501,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
501 def apply_weights(self, x: torch.Tensor,
502 bias: Optional[torch.Tensor]) -> torch.Tensor:
503 output = self.base_layer.linear_method.apply_weights(
508 - self.base_layer.linear_weights, x, bias)
504 + self.base_layer, x, bias)
505 _apply_lora_packed_nslice(
506 x,
507 self.lora_a_stacked,
@@ -746,7 +742,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
742 def apply_weights(self, x: torch.Tensor,
743 bias: Optional[torch.Tensor]) -> torch.Tensor:
744 output = self.base_layer.linear_method.apply_weights(
749 - self.base_layer.linear_weights, x, bias)
745 + self.base_layer, x, bias)
746 _apply_lora_packed_nslice(
747 x,
748 self.lora_a_stacked,
@@ -838,7 +834,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
834
835 def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
836 output = self.base_layer.linear_method.apply_weights(
841 - self.base_layer.linear_weights, x)
837 + self.base_layer, x)
838 _apply_lora(
839 x,
840 self.lora_a_stacked,
vllm/model_executor/layers/linear.py
+40 -37
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Any, Dict, List, Optional
2 +from typing import List, Optional
3
4 import torch
5 import torch.nn.functional as F
@@ -28,19 +28,24 @@ class LinearMethodBase(ABC):
28 """Base class for different (maybe quantized) linear methods."""
29
30 @abstractmethod
31 - def create_weights(self, input_size_per_partition: int,
31 + def create_weights(self, layer: torch.nn.Module,
32 + input_size_per_partition: int,
33 output_size_per_partition: int, input_size: int,
33 - output_size: int,
34 - params_dtype: torch.dtype) -> Dict[str, Any]:
35 - """Create weights for a linear layer."""
34 + output_size: int, params_dtype: torch.dtype,
35 + **extra_weight_attrs):
36 + """Create weights for a linear layer.
37 +
38 + The weights will be set as attributes of the layer."""
39 raise NotImplementedError
40
41 @abstractmethod
42 def apply_weights(self,
40 - weights: Dict[str, torch.Tensor],
43 + layer: torch.nn.Module,
44 x: torch.Tensor,
45 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
43 - """Apply the weights to the input tensor."""
46 + """Apply the weights in layer to the input tensor.
47 +
48 + Expects create_weights to have been called before on the layer."""
49 raise NotImplementedError
50
51
@@ -55,22 +60,24 @@ class UnquantizedLinearMethod(LinearMethodBase):
60 def __init__(self, separate_bias_add: bool = False):
61 self.separate_bias_add = separate_bias_add
62
58 - def create_weights(self, input_size_per_partition: int,
63 + def create_weights(self, layer: torch.nn.Module,
64 + input_size_per_partition: int,
65 output_size_per_partition: int, input_size: int,
60 - output_size: int,
61 - params_dtype: torch.dtype) -> Dict[str, Any]:
66 + output_size: int, params_dtype: torch.dtype,
67 + **extra_weight_attrs):
68 weight = Parameter(torch.empty(output_size_per_partition,
69 input_size_per_partition,
70 dtype=params_dtype),
71 requires_grad=False)
72 set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
67 - return {"weight": weight}
73 + layer.register_parameter("weight", weight)
74 + set_weight_attrs(weight, extra_weight_attrs)
75
76 def apply_weights(self,
70 - weights: Dict[str, torch.Tensor],
77 + layer: torch.nn.Module,
78 x: torch.Tensor,
79 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
73 - weight = weights["weight"]
80 + weight = layer.weight
81 if self.separate_bias_add:
82 if bias is not None:
83 return F.linear(x, weight) + bias
@@ -111,12 +118,9 @@ class ReplicatedLinear(torch.nn.Module):
118 if linear_method is None:
119 linear_method = UnquantizedLinearMethod()
120 self.linear_method = linear_method
114 - self.linear_weights = self.linear_method.create_weights(
115 - self.input_size, self.output_size, self.input_size,
116 - self.output_size, self.params_dtype)
117 - for name, weight in self.linear_weights.items():
118 - if isinstance(weight, torch.Tensor):
119 - self.register_parameter(name, weight)
121 + self.linear_method.create_weights(self, self.input_size,
122 + self.output_size, self.input_size,
123 + self.output_size, self.params_dtype)
124 if bias:
125 self.bias = Parameter(
126 torch.empty(self.output_size, dtype=self.params_dtype))
@@ -126,7 +130,7 @@ class ReplicatedLinear(torch.nn.Module):
130
131 def forward(self, x: torch.Tensor) -> torch.Tensor:
132 bias = self.bias if not self.skip_bias_add else None
129 - output = self.linear_method.apply_weights(self.linear_weights, x, bias)
133 + output = self.linear_method.apply_weights(self, x, bias)
134 output_bias = self.bias if self.skip_bias_add else None
135 return output, output_bias
136
@@ -177,13 +181,13 @@ class ColumnParallelLinear(torch.nn.Module):
181 if linear_method is None:
182 linear_method = UnquantizedLinearMethod()
183 self.linear_method = linear_method
180 - self.linear_weights = self.linear_method.create_weights(
181 - self.input_size, self.output_size_per_partition, self.input_size,
182 - self.output_size, self.params_dtype)
183 - for name, weight in self.linear_weights.items():
184 - if isinstance(weight, torch.Tensor):
185 - self.register_parameter(name, weight)
186 - set_weight_attrs(weight, {"weight_loader": self.weight_loader})
184 + self.linear_method.create_weights(self,
185 + self.input_size,
186 + self.output_size_per_partition,
187 + self.input_size,
188 + self.output_size,
189 + self.params_dtype,
190 + weight_loader=self.weight_loader)
191 if bias:
192 self.bias = Parameter(
193 torch.empty(self.output_size_per_partition,
@@ -211,8 +215,7 @@ class ColumnParallelLinear(torch.nn.Module):
215 bias = self.bias if not self.skip_bias_add else None
216
217 # Matrix multiply.
214 - output_parallel = self.linear_method.apply_weights(
215 - self.linear_weights, input_, bias)
218 + output_parallel = self.linear_method.apply_weights(self, input_, bias)
219 if self.gather_output:
220 # All-gather across the partitions.
221 output = tensor_model_parallel_all_gather(output_parallel)
@@ -523,13 +526,13 @@ class RowParallelLinear(torch.nn.Module):
526 if linear_method is None:
527 linear_method = UnquantizedLinearMethod()
528 self.linear_method = linear_method
526 - self.linear_weights = self.linear_method.create_weights(
527 - self.input_size_per_partition, self.output_size, self.input_size,
528 - self.output_size, self.params_dtype)
529 - for name, weight in self.linear_weights.items():
530 - if isinstance(weight, torch.Tensor):
531 - self.register_parameter(name, weight)
532 - set_weight_attrs(weight, {"weight_loader": self.weight_loader})
529 + self.linear_method.create_weights(self,
530 + self.input_size_per_partition,
531 + self.output_size,
532 + self.input_size,
533 + self.output_size,
534 + self.params_dtype,
535 + weight_loader=self.weight_loader)
536
537 if not reduce_results and (bias and not skip_bias_add):
538 raise ValueError("When not reduce the results, adding bias to the "
@@ -569,7 +572,7 @@ class RowParallelLinear(torch.nn.Module):
572
573 # Matrix multiply.
574 output_parallel = self.linear_method.apply_weights(
572 - self.linear_weights, input_parallel)
575 + self, input_parallel)
576 if self.reduce_results and self.tp_size > 1:
577 output_ = tensor_model_parallel_all_reduce(output_parallel)
578 else:
vllm/model_executor/layers/quantization/awq.py
+16 -13
@@ -79,10 +79,11 @@ class AWQLinearMethod(LinearMethodBase):
79 def __init__(self, quant_config: AWQConfig):
80 self.quant_config = quant_config
81
82 - def create_weights(self, input_size_per_partition: int,
82 + def create_weights(self, layer: torch.nn.Module,
83 + input_size_per_partition: int,
84 output_size_per_partition: int, input_size: int,
84 - output_size: int,
85 - params_dtype: torch.dtype) -> Dict[str, Any]:
85 + output_size: int, params_dtype: torch.dtype,
86 + **extra_weight_attrs):
87 if input_size_per_partition % self.quant_config.group_size != 0:
88 raise ValueError(
89 "The input size is not aligned with the quantized "
@@ -136,19 +137,21 @@ class AWQLinearMethod(LinearMethodBase):
137 "input_dim": 0,
138 "output_dim": 1,
139 })
139 - return {
140 - "qweight": qweight,
141 - "qzeros": qzeros,
142 - "scales": scales,
143 - }
140 +
141 + layer.register_parameter("qweight", qweight)
142 + set_weight_attrs(qweight, extra_weight_attrs)
143 + layer.register_parameter("qzeros", qzeros)
144 + set_weight_attrs(qzeros, extra_weight_attrs)
145 + layer.register_parameter("scales", scales)
146 + set_weight_attrs(scales, extra_weight_attrs)
147
148 def apply_weights(self,
146 - weights: Dict[str, Any],
149 + layer: torch.nn.Module,
150 x: torch.Tensor,
151 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
149 - qweight = weights["qweight"]
150 - scales = weights["scales"]
151 - qzeros = weights["qzeros"]
152 + qweight = layer.qweight
153 + scales = layer.scales
154 + qzeros = layer.qzeros
155 pack_factor = self.quant_config.pack_factor
156 out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, ))
157 reshaped_x = x.reshape(-1, x.shape[-1])
@@ -163,5 +166,5 @@ class AWQLinearMethod(LinearMethodBase):
166 out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros,
167 pack_factor)
168 if bias is not None:
166 - out = out + bias
169 + out.add_(bias)
170 return out.reshape(out_shape)
vllm/model_executor/layers/quantization/gptq.py
+26 -21
@@ -89,12 +89,14 @@ class GPTQLinearMethod(LinearMethodBase):
89
90 def create_weights(
91 self,
92 + layer: torch.nn.Module,
93 input_size_per_partition: int,
94 output_size_per_partition: int,
95 input_size: int,
96 output_size: int,
97 params_dtype: torch.dtype,
97 - ) -> Dict[str, Any]:
98 + **extra_weight_attrs,
99 + ):
100 del output_size # Unused.
101 if input_size_per_partition % self.quant_config.group_size != 0:
102 raise ValueError(
@@ -179,37 +181,40 @@ class GPTQLinearMethod(LinearMethodBase):
181 "input_dim": scale_and_zero_input_dim,
182 "output_dim": 1,
183 })
182 - return {
183 - "qweight": qweight,
184 - "g_idx": g_idx,
185 - "qzeros": qzeros,
186 - "scales": scales,
187 - "exllama_state": exllama_state,
188 - }
184 +
185 + layer.register_parameter("qweight", qweight)
186 + set_weight_attrs(qweight, extra_weight_attrs)
187 + layer.register_parameter("g_idx", g_idx)
188 + set_weight_attrs(g_idx, extra_weight_attrs)
189 + layer.register_parameter("qzeros", qzeros)
190 + set_weight_attrs(qzeros, extra_weight_attrs)
191 + layer.register_parameter("scales", scales)
192 + set_weight_attrs(scales, extra_weight_attrs)
193 +
194 + layer.exllama_state = exllama_state
195
196 def apply_weights(self,
191 - weights: Dict[str, Any],
197 + layer: torch.nn.Module,
198 x: torch.Tensor,
199 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
194 - qweight = weights["qweight"]
200 + qweight = layer.qweight
201 out_shape = x.shape[:-1] + (qweight.shape[-1], )
202 reshaped_x = x.reshape(-1, x.shape[-1])
203 # exllama needs to shuffle the weight after the weight is loaded
204 # here we do the shuffle on first forward pass
199 - if weights["exllama_state"] == ExllamaState.UNINITIALIZED:
205 + if layer.exllama_state == ExllamaState.UNINITIALIZED:
206 if self.quant_config.desc_act:
201 - weights["g_idx"] = torch.argsort(weights["g_idx"]).to(
202 - torch.int)
207 + layer.g_idx.data = torch.argsort(layer.g_idx).to(torch.int)
208 else:
204 - weights["g_idx"] = torch.empty((1, 1), device="meta")
205 - weights["exllama_state"] = ExllamaState.READY
206 - ops.gptq_shuffle(weights["qweight"], weights["g_idx"],
209 + layer.g_idx.data = torch.empty((0, ),
210 + device=layer.g_idx.device)
211 + layer.exllama_state = ExllamaState.READY
212 + ops.gptq_shuffle(layer.qweight, layer.g_idx,
213 self.quant_config.weight_bits)
208 - output = ops.gptq_gemm(reshaped_x, weights["qweight"],
209 - weights["qzeros"], weights["scales"],
210 - weights["g_idx"],
211 - weights["exllama_state"] == ExllamaState.READY,
214 + output = ops.gptq_gemm(reshaped_x, layer.qweight, layer.qzeros,
215 + layer.scales, layer.g_idx,
216 + layer.exllama_state == ExllamaState.READY,
217 self.quant_config.weight_bits)
218 if bias is not None:
214 - output = output + bias
219 + output.add_(bias)
220 return output.reshape(out_shape)
vllm/model_executor/layers/quantization/marlin.py
+13 -10
@@ -91,12 +91,14 @@ class MarlinLinearMethod(LinearMethodBase):
91
92 def create_weights(
93 self,
94 + layer: torch.nn.Module,
95 input_size_per_partition: int,
96 output_size_per_partition: int,
97 input_size: int,
98 output_size: int,
99 params_dtype: torch.dtype,
99 - ) -> Dict[str, Any]:
100 + **extra_weight_attrs,
101 + ):
102 del output_size # Unused.
103
104 if params_dtype != torch.float16:
@@ -187,21 +189,22 @@ class MarlinLinearMethod(LinearMethodBase):
189 dtype=torch.int),
190 requires_grad=False)
191
190 - return {
191 - "B": qweight,
192 - "s": scales,
193 - "workspace": workspace,
194 - }
192 + layer.register_parameter("B", qweight)
193 + set_weight_attrs(qweight, extra_weight_attrs)
194 + layer.register_parameter("s", scales)
195 + set_weight_attrs(scales, extra_weight_attrs)
196 + layer.register_parameter("workspace", workspace)
197 + set_weight_attrs(workspace, extra_weight_attrs)
198
199 def apply_weights(
200 self,
198 - weights: Dict[str, Any],
201 + layer: torch.nn.Module,
202 x: torch.Tensor,
203 bias: Optional[torch.Tensor] = None,
204 ) -> torch.Tensor:
202 - qweight = weights["B"]
203 - scales = weights["s"]
204 - workspace = weights["workspace"]
205 + qweight = layer.B
206 + scales = layer.s
207 + workspace = layer.workspace
208
209 x_2d = x.view(-1, x.shape[-1])
210
vllm/model_executor/layers/quantization/squeezellm.py
+13 -11
@@ -68,10 +68,11 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
68 def __init__(self, quant_config: SqueezeLLMConfig):
69 self.quant_config = quant_config
70
71 - def create_weights(self, input_size_per_partition: int,
71 + def create_weights(self, layer: torch.nn.Module,
72 + input_size_per_partition: int,
73 output_size_per_partition: int, input_size: int,
73 - output_size: int,
74 - params_dtype: torch.dtype) -> Dict[str, Any]:
74 + output_size: int, params_dtype: torch.dtype,
75 + **extra_weight_attrs):
76 if input_size_per_partition % self.quant_config.pack_factor != 0:
77 raise ValueError(
78 "The input size is not aligned with the quantized "
@@ -103,17 +104,18 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
104 set_weight_attrs(lookup_table, {
105 "output_dim": 0,
106 })
106 - return {
107 - "qweight": qweight,
108 - "lookup_table": lookup_table,
109 - }
107 +
108 + layer.register_parameter("qweight", qweight)
109 + set_weight_attrs(qweight, extra_weight_attrs)
110 + layer.register_parameter("lookup_table", lookup_table)
111 + set_weight_attrs(lookup_table, extra_weight_attrs)
112
113 def apply_weights(self,
112 - weights: Dict[str, Any],
114 + layer: torch.nn.Module,
115 x: torch.Tensor,
116 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
115 - qweight = weights["qweight"]
116 - lookup_table = weights["lookup_table"]
117 + qweight = layer.qweight
118 + lookup_table = layer.lookup_table
119 out_shape = x.shape[:-1] + (qweight.shape[-1], )
120 reshaped_x = x.reshape(-1, x.shape[-1])
121 if is_hip():
@@ -126,5 +128,5 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
128 ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
129
130 if bias is not None:
129 - out = out + bias
131 + out.add_(bias)
132 return out.reshape(out_shape)