@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f1df5dbfd6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc] Update `marlin` to use vLLMParameters (#7803)

Dipika Sikka committed Aug 23, 2024 at 14:30 UTC f1df5dbfd6782408228f39bdc0722fa465629f0f
3 files changed +41 -34
tests/weight_loading/models.txt
+3 -1
@@ -15,4 +15,6 @@ compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15 compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
16 awq, casperhansen/mixtral-instruct-awq, main
17 awq_marlin, casperhansen/mixtral-instruct-awq, main
18 -fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
\ No newline at end of file
18 +fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
19 +marlin, nm-testing/zephyr-beta-7b-marlin-g128, main
20 +marlin, robertgshaw2/zephyr-7b-beta-channelwise-marlin, main
\ No newline at end of file
vllm/model_executor/layers/linear.py
+2 -1
@@ -22,7 +22,8 @@ logger = init_logger(__name__)
22
23 WEIGHT_LOADER_V2_SUPPORTED = [
24 "CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
25 - "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod"
25 + "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
26 + "MarlinLinearMethod"
27 ]
28
29
vllm/model_executor/layers/quantization/marlin.py
+36 -32
@@ -9,7 +9,10 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9 from vllm.model_executor.layers.quantization.base_config import (
10 QuantizationConfig)
11 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
12 -from vllm.model_executor.utils import set_weight_attrs
12 +from vllm.model_executor.parameter import (BasevLLMParameter,
13 + ChannelQuantScaleParameter,
14 + GroupQuantScaleParameter,
15 + PackedvLLMParameter)
16
17 logger = init_logger(__name__)
18
@@ -132,6 +135,7 @@ class MarlinLinearMethod(LinearMethodBase):
135 **extra_weight_attrs,
136 ):
137 del output_size # Unused.
138 + weight_loader = extra_weight_attrs["weight_loader"]
139
140 if params_dtype != torch.float16:
141 raise ValueError(
@@ -170,64 +174,64 @@ class MarlinLinearMethod(LinearMethodBase):
174 "Each permutation group must reside on the same gpu")
175
176 # Quantized 4Bit weights packed into Int32.
173 - qweight = Parameter(
174 - torch.empty(
177 + qweight = PackedvLLMParameter(
178 + data=torch.empty(
179 input_size_per_partition // self.quant_config.tile_size,
180 output_size_per_partition * self.quant_config.tile_size //
181 self.quant_config.pack_factor,
182 device="cuda",
183 dtype=torch.int32,
184 ),
181 - requires_grad=False,
182 - )
183 - set_weight_attrs(
184 - qweight,
185 - {
186 - "input_dim": 0,
187 - "output_dim": 1,
188 - "packed_dim": 1,
189 - "pack_factor": self.quant_config.pack_factor,
190 - "marlin_tile_size": self.quant_config.tile_size,
191 - },
192 - )
185 + input_dim=0,
186 + output_dim=1,
187 + packed_dim=1,
188 + packed_factor=self.quant_config.pack_factor,
189 + marlin_tile_size=self.quant_config.tile_size,
190 + weight_loader=weight_loader)
191
192 # Determine if channelwise or not
193 input_groups = (1 if self.quant_config.group_size == -1 else
194 input_size_per_partition //
195 self.quant_config.group_size)
196
199 - scales = Parameter(
197 + weight_scale_args = {
198 + "data":
199 torch.empty(
200 input_groups,
201 output_size_per_partition,
202 device="cuda",
203 dtype=params_dtype,
204 ),
206 - requires_grad=False,
207 - )
208 - set_weight_attrs(
209 - scales,
210 - {
211 - "input_dim": None if input_groups == 1 else 0,
212 - "output_dim": 1,
213 - },
214 - )
205 + "weight_loader":
206 + weight_loader
207 + }
208 + if input_groups == 1:
209 + scales = ChannelQuantScaleParameter(output_dim=1,
210 + **weight_scale_args)
211 + else:
212 + scales = GroupQuantScaleParameter(output_dim=1,
213 + input_dim=0,
214 + **weight_scale_args)
215
216 # Allocate workspace (Used for internal locking mechanism)
217 max_workspace_size = (
218 output_size_per_partition //
219 self.quant_config.min_n_threads) * self.quant_config.max_parallel
220 - workspace = Parameter(torch.zeros(max_workspace_size,
221 - device="cuda",
222 - dtype=torch.int),
223 - requires_grad=False)
220 +
221 + workspace = BasevLLMParameter(data=torch.zeros(max_workspace_size,
222 + device="cuda",
223 + dtype=torch.int),
224 + weight_loader=weight_loader)
225
226 layer.register_parameter("B", qweight)
226 - set_weight_attrs(qweight, extra_weight_attrs)
227 layer.register_parameter("s", scales)
228 - set_weight_attrs(scales, extra_weight_attrs)
228 layer.register_parameter("workspace", workspace)
230 - set_weight_attrs(workspace, extra_weight_attrs)
229 +
230 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
231 + # required by torch.compile
232 + layer.B = Parameter(layer.B.data, requires_grad=False)
233 + layer.s = Parameter(layer.s.data, requires_grad=False)
234 + layer.workspace = Parameter(layer.workspace.data, requires_grad=False)
235
236 def apply(
237 self,