@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fb377d7e74
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Update `gptq_marlin` to use new vLLMParameters (#7281)

Dipika Sikka committed Aug 13, 2024 at 14:30 UTC fb377d7e74228d477e270d8a8e53410db29ed755
8 files changed +225 -89
.buildkite/test-pipeline.yaml
+10
@@ -314,6 +314,16 @@ steps:
314 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
315 - pytest -v -s -x lora/test_long_context.py
316
317 +- label: Weight Loading Multiple GPU Test
318 + working_dir: "/vllm-workspace/tests"
319 + num_gpus: 2
320 + source_file_dependencies:
321 + - vllm/
322 + - tests/weight_loading
323 + commands:
324 + - bash weight_loading/run_model_weight_loading_test.sh
325 +
326 +
327 ##### multi gpus test #####
328 ##### A100 test #####
329
tests/weight_loading/models.txt new
+15
@@ -0,0 +1,15 @@
1 +gptq_marlin, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
2 +gptq_marlin, TheBloke/Llama-2-7B-GPTQ, main
3 +gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
4 +gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
5 +gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
6 +gptq_marlin, TechxGenus/gemma-1.1-2b-it-GPTQ, main
7 +compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main
8 +compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main
9 +compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main
10 +compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2, main
11 +compressed-tensors, nm-testing/tinyllama-oneshot-w4a16-group128-v2, main
12 +compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main
13 +compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
14 +compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15 +compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
\ No newline at end of file
tests/weight_loading/run_model_weight_loading_test.sh new
+32
@@ -0,0 +1,32 @@
1 +#!/bin/bash
2 +SUCCESS=0
3 +
4 +IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "weight_loading/models.txt"
5 +
6 +for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
7 +do
8 + LOCAL_SUCCESS=0
9 + IFS=', ' read -r -a array <<< "$MODEL_CONFIG"
10 +
11 + echo "=== RUNNING MODEL: $MODEL_CONFIG ==="
12 +
13 + export QUANTIZATION=${array[0]}
14 + export MODEL_NAME=${array[1]}
15 + export REVISION=${array[2]}
16 + pytest -s weight_loading/test_weight_loading.py || LOCAL_SUCCESS=$?
17 +
18 + if [[ $LOCAL_SUCCESS == 0 ]]; then
19 + echo "=== PASSED MODEL: ${MODEL_CONFIG} ==="
20 + else
21 + echo "=== FAILED MODEL: ${MODEL_CONFIG} ==="
22 + fi
23 +
24 + SUCCESS=$((SUCCESS + LOCAL_SUCCESS))
25 +
26 +done
27 +
28 +if [ "${SUCCESS}" -eq "0" ]; then
29 + exit 0
30 +else
31 + exit 1
32 +fi
tests/weight_loading/test_weight_loading.py new
+20
@@ -0,0 +1,20 @@
1 +import os
2 +
3 +MAX_MODEL_LEN = 1024
4 +MODEL_NAME = os.environ.get("MODEL_NAME",
5 + "robertgshaw2/zephyr-7b-beta-channelwise-gptq")
6 +REVISION = os.environ.get("REVISION", "main")
7 +QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin")
8 +
9 +
10 +def test_weight_loading(vllm_runner):
11 + with vllm_runner(model_name=MODEL_NAME,
12 + revision=REVISION,
13 + dtype="auto",
14 + quantization=QUANTIZATION,
15 + max_model_len=MAX_MODEL_LEN,
16 + tensor_parallel_size=2) as model:
17 +
18 + output = model.generate_greedy("Hello world!", max_tokens=20)
19 + print(output)
20 + assert output
vllm/model_executor/layers/linear.py
+3 -1
@@ -20,7 +20,9 @@ from vllm.model_executor.utils import set_weight_attrs
20
21 logger = init_logger(__name__)
22
23 -WEIGHT_LOADER_V2_SUPPORTED = ["CompressedTensorsLinearMethod"]
23 +WEIGHT_LOADER_V2_SUPPORTED = [
24 + "CompressedTensorsLinearMethod", "GPTQMarlinLinearMethod"
25 +]
26
27
28 def adjust_marlin_shard(param, shard_size, shard_offset):
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+1 -1
@@ -105,7 +105,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
105 dtype=params_dtype,
106 )
107 }
108 - if self.group_size == -1:
108 + if not partition_scales:
109 weight_scale = ChannelQuantScaleParameter(output_dim=0,
110 **weight_scale_args)
111 else:
vllm/model_executor/layers/quantization/gptq_marlin.py
+59 -62
@@ -1,12 +1,11 @@
1 from typing import Any, Dict, List, Optional
2
3 import torch
4 -from torch.nn.parameter import Parameter
4 +from torch.nn import Parameter
5
6 from vllm import _custom_ops as ops
7 from vllm.logger import init_logger
8 -from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9 - set_weight_attrs)
8 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9 from vllm.model_executor.layers.quantization.base_config import (
10 QuantizationConfig)
11 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
@@ -15,6 +14,11 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
14 marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
15 verify_marlin_supported, verify_marlin_supports_shape)
16 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
17 +from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
18 + GroupQuantScaleParameter,
19 + PackedColumnParameter,
20 + PackedvLLMParameter,
21 + RowvLLMParameter)
22 from vllm.scalar_type import scalar_types
23
24 logger = init_logger(__name__)
@@ -159,9 +163,11 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
163 params_dtype: torch.dtype,
164 **extra_weight_attrs,
165 ) -> None:
166 +
167 del output_size
168 output_size_per_partition = sum(output_partition_sizes)
169 is_row_parallel = input_size != input_size_per_partition
170 + weight_loader = extra_weight_attrs.get("weight_loader")
171
172 # Normalize group_size
173 if self.quant_config.group_size != -1:
@@ -190,79 +196,66 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
196 scales_and_zp_size = input_size_per_partition // group_size
197
198 # Quantized weights
193 - qweight = Parameter(
194 - torch.empty(
199 + qweight = PackedvLLMParameter(
200 + data=torch.empty(
201 input_size_per_partition // self.quant_config.pack_factor,
202 output_size_per_partition,
203 dtype=torch.int32,
204 ),
199 - requires_grad=False,
200 - )
201 - set_weight_attrs(
202 - qweight,
203 - {
204 - **extra_weight_attrs,
205 - "input_dim": 0,
206 - "output_dim": 1,
207 - "packed_dim": 0,
208 - "pack_factor": self.quant_config.pack_factor,
209 - },
210 - )
205 + input_dim=0,
206 + output_dim=1,
207 + packed_dim=0,
208 + packed_factor=self.quant_config.pack_factor,
209 + weight_loader=weight_loader)
210
211 # Activation order
213 - g_idx = Parameter(
212 + g_idx = RowvLLMParameter(data=torch.empty(
213 + input_size_per_partition,
214 + dtype=torch.int32,
215 + ),
216 + input_dim=0,
217 + weight_loader=weight_loader)
218 +
219 + qzeros_args = {
220 + "data":
221 torch.empty(
215 - input_size_per_partition,
222 + scales_and_zp_size,
223 + output_size_per_partition // self.quant_config.pack_factor,
224 dtype=torch.int32,
225 ),
218 - requires_grad=False,
219 - )
220 - # Ignore warning from fused linear layers such as QKVParallelLinear.
221 - set_weight_attrs(
222 - g_idx,
223 - {
224 - **extra_weight_attrs, "input_dim": 0,
225 - "ignore_warning": True
226 - },
227 - )
228 -
229 - # Scales
230 - scales = Parameter(
226 + "weight_loader":
227 + weight_loader
228 + }
229 + weight_scale_args = {
230 + "data":
231 torch.empty(
232 scales_and_zp_size,
233 output_size_per_partition,
234 dtype=params_dtype,
235 ),
236 - requires_grad=False,
237 - )
238 - set_weight_attrs(
239 - scales,
240 - {
241 - **extra_weight_attrs,
242 - "input_dim": scales_and_zp_input_dim,
243 - "output_dim": 1,
244 - },
245 - )
246 -
247 - # Quantized zero-points
248 - qzeros = Parameter(
249 - torch.empty(
250 - scales_and_zp_size,
251 - output_size_per_partition // self.quant_config.pack_factor,
252 - dtype=torch.int32,
253 - ),
254 - requires_grad=False,
255 - )
256 - set_weight_attrs(
257 - qzeros,
258 - {
259 - **extra_weight_attrs,
260 - "input_dim": scales_and_zp_input_dim,
261 - "output_dim": 1,
262 - "packed_dim": 1,
263 - "pack_factor": self.quant_config.pack_factor,
264 - },
265 - )
236 + "weight_loader":
237 + weight_loader
238 + }
239 +
240 + if scales_and_zp_input_dim is None:
241 + scales = ChannelQuantScaleParameter(output_dim=1,
242 + **weight_scale_args)
243 + qzeros = PackedColumnParameter(
244 + output_dim=1,
245 + packed_dim=1,
246 + packed_factor=self.quant_config.pack_factor,
247 + **qzeros_args)
248 +
249 + else:
250 + scales = GroupQuantScaleParameter(output_dim=1,
251 + input_dim=0,
252 + **weight_scale_args)
253 + qzeros = PackedvLLMParameter(
254 + input_dim=0,
255 + output_dim=1,
256 + packed_dim=1,
257 + packed_factor=self.quant_config.pack_factor,
258 + **qzeros_args)
259
260 layer.register_parameter("qweight", qweight)
261 layer.register_parameter("g_idx", g_idx)
@@ -280,6 +273,10 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
273 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
274 device = layer.qweight.device
275
276 + # required by torch.compile
277 + layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
278 + layer.scales = Parameter(layer.scales.data, requires_grad=False)
279 +
280 # Allocate marlin workspace
281 layer.workspace = marlin_make_workspace(
282 layer.output_size_per_partition, device)
vllm/model_executor/parameter.py
+85 -25
@@ -9,7 +9,7 @@ from vllm.logger import init_logger
9 __all__ = [
10 "BasevLLMParameter", "PackedvLLMParameter", "PerTensorScaleParameter",
11 "ModelWeightParameter", "ChannelQuantScaleParameter",
12 - "GroupQuantScaleParameter"
12 + "GroupQuantScaleParameter", "PackedColumnParameter", "RowvLLMParameter"
13 ]
14
15 logger = init_logger(__name__)
@@ -92,7 +92,8 @@ class _ColumnvLLMParameter(BasevLLMParameter):
92 shard_size = kwargs.get("shard_size")
93 if isinstance(
94 self,
95 - PackedvLLMParameter) and self.packed_dim == self.output_dim:
95 + (PackedColumnParameter,
96 + PackedvLLMParameter)) and self.packed_dim == self.output_dim:
97 shard_size, shard_offset = self.adjust_shard_indexes_for_packing(
98 shard_offset=shard_offset, shard_size=shard_size)
99
@@ -115,7 +116,8 @@ class _ColumnvLLMParameter(BasevLLMParameter):
116
117 if isinstance(
118 self,
118 - PackedvLLMParameter) and self.output_dim == self.packed_dim:
119 + (PackedColumnParameter,
120 + PackedvLLMParameter)) and self.output_dim == self.packed_dim:
121 shard_size, shard_offset = self.adjust_shard_indexes_for_packing(
122 shard_offset=shard_offset, shard_size=shard_size)
123
@@ -131,12 +133,12 @@ class _ColumnvLLMParameter(BasevLLMParameter):
133 param_data.copy_(loaded_weight)
134
135
134 -class ModelWeightParameter(_ColumnvLLMParameter):
136 +class RowvLLMParameter(BasevLLMParameter):
137 """
136 - Parameter class for linear layer weights. Extends the
137 - _ColumnvLLMParameter by adding loading functionality
138 - for linear layers with row parallel functionality.
139 - Requires an input dimension to be defined.
138 + Parameter class defining weight_loading functionality
139 + (load_row_parallel_weight) for parameters being loaded
140 + into linear layers with row parallel functionality.
141 + Requires an input_dim to be defined.
142 """
143
144 def __init__(self, input_dim: int, **kwargs):
@@ -160,10 +162,18 @@ class ModelWeightParameter(_ColumnvLLMParameter):
162 self.data.copy_(loaded_weight)
163
164
163 -class GroupQuantScaleParameter(ModelWeightParameter):
165 +class ModelWeightParameter(_ColumnvLLMParameter, RowvLLMParameter):
166 + """
167 + Parameter class for linear layer weights. Uses both column and
168 + row parallelism.
169 + """
170 + pass
171 +
172 +
173 +class GroupQuantScaleParameter(_ColumnvLLMParameter, RowvLLMParameter):
174 """
175 Parameter class for weight scales loaded for weights with
166 - grouped quantization. Equivalent to ModelWeightParameter.
176 + grouped quantization. Uses both column and row parallelism.
177 """
178 pass
179
@@ -171,7 +181,7 @@ class GroupQuantScaleParameter(ModelWeightParameter):
181 class ChannelQuantScaleParameter(_ColumnvLLMParameter):
182 """
183 Parameter class for weight scales loaded for weights with
174 - channel-wise quantization. Equivalent to _ColumnvLLMParameter.
184 + channel-wise quantization. Equivalent to _ColumnvLLMParameter.
185 """
186 pass
187
@@ -181,7 +191,7 @@ class PerTensorScaleParameter(BasevLLMParameter):
191 Parameter class for scales where the number of scales is
192 equivalent to the number of logical matrices in fused linear
193 layers (e.g. for QKV, there are 3 scales loaded from disk).
184 - This is relevant to weights with per-tensor quantization.
194 + This is relevant to weights with per-tensor quantization.
195 Adds functionality to map the scalers to a shard during
196 weight loading.
197
@@ -232,6 +242,43 @@ class PerTensorScaleParameter(BasevLLMParameter):
242 param_data.copy_(loaded_weight)
243
244
245 +class PackedColumnParameter(_ColumnvLLMParameter):
246 + """
247 + Parameter for model parameters which are packed on disk
248 + and support column parallelism only. See PackedvLLMParameter
249 + for more details on the packed properties.
250 + """
251 +
252 + def __init__(self,
253 + packed_factor: int,
254 + packed_dim: int,
255 + marlin_tile_size: Optional[int] = None,
256 + **kwargs):
257 + self._packed_factor = packed_factor
258 + self._packed_dim = packed_dim
259 + self._marlin_tile_size = marlin_tile_size
260 + super().__init__(**kwargs)
261 +
262 + @property
263 + def packed_dim(self):
264 + return self._packed_dim
265 +
266 + @property
267 + def packed_factor(self):
268 + return self._packed_factor
269 +
270 + @property
271 + def marlin_tile_size(self):
272 + return self._marlin_tile_size
273 +
274 + def adjust_shard_indexes_for_packing(self, shard_size, shard_offset):
275 + return _adjust_shard_indexes_for_packing(
276 + shard_size=shard_size,
277 + shard_offset=shard_offset,
278 + packed_factor=self.packed_factor,
279 + marlin_tile_size=self.marlin_tile_size)
280 +
281 +
282 class PackedvLLMParameter(ModelWeightParameter):
283 """
284 Parameter for model weights which are packed on disk.
@@ -239,7 +286,7 @@ class PackedvLLMParameter(ModelWeightParameter):
286 Extends the ModelWeightParameter to take in the
287 packed factor, the packed dimension, and optionally, marlin
288 tile size for marlin kernels. Adjusts the shard_size and
242 - shard_offset for fused linear layers model weight loading
289 + shard_offset for fused linear layers model weight loading
290 by accounting for packing and optionally, marlin tile size.
291 """
292
@@ -250,7 +297,7 @@ class PackedvLLMParameter(ModelWeightParameter):
297 **kwargs):
298 self._packed_factor = packed_factor
299 self._packed_dim = packed_dim
253 - self._marlin_tile = marlin_tile_size
300 + self._marlin_tile_size = marlin_tile_size
301 super().__init__(**kwargs)
302
303 @property
@@ -262,16 +309,29 @@ class PackedvLLMParameter(ModelWeightParameter):
309 return self._packed_factor
310
311 @property
265 - def marlin_tile(self):
266 - return self._marlin_tile
267 -
268 - def _adjust_shard_indexes_for_marlin(self, shard_size, shard_offset):
269 - return shard_size * self.marlin_tile, shard_offset * self.marlin_tile
312 + def marlin_tile_size(self):
313 + return self._marlin_tile_size
314
315 def adjust_shard_indexes_for_packing(self, shard_size, shard_offset):
272 - shard_size = shard_size // self.packed_factor
273 - shard_offset = shard_offset // self.packed_factor
274 - if self.marlin_tile is not None:
275 - return self._adjust_shard_indexes_for_marlin(
276 - shard_size, shard_offset)
277 - return shard_size, shard_offset
316 + return _adjust_shard_indexes_for_packing(
317 + shard_size=shard_size,
318 + shard_offset=shard_offset,
319 + packed_factor=self.packed_factor,
320 + marlin_tile_size=self.marlin_tile_size)
321 +
322 +
323 +def _adjust_shard_indexes_for_marlin(shard_size, shard_offset,
324 + marlin_tile_size):
325 + return shard_size * marlin_tile_size, shard_offset * marlin_tile_size
326 +
327 +
328 +def _adjust_shard_indexes_for_packing(shard_size, shard_offset, packed_factor,
329 + marlin_tile_size):
330 + shard_size = shard_size // packed_factor
331 + shard_offset = shard_offset // packed_factor
332 + if marlin_tile_size is not None:
333 + return _adjust_shard_indexes_for_marlin(
334 + shard_size=shard_size,
335 + shard_offset=shard_offset,
336 + marlin_tile_size=marlin_tile_size)
337 + return shard_size, shard_offset