@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / af9ad46fca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ Misc ] Refactor w8a8 to use `process_weights_after_load` (Simplify Weight Loading) (#5940)

Co-authored-by: Robert Shaw <rshaw@neuralmagic>

Robert Shaw committed Jun 30, 2024 at 19:06 UTC af9ad46fca6e594797b83e5ecb2e1f31ca5e9fac
10 files changed +153 -158
tests/quantization/test_compressed_tensors.py
+19 -9
@@ -11,14 +11,18 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
11 CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
12 CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
13 CompressedTensorsWNA16)
14 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 + QuantizationType)
16
17
18 @pytest.mark.parametrize("model_args", [
17 - ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", "tensor"),
18 - ("nm-testing/tinyllama-oneshot-w8-channel-a8-tensor", "channel"),
19 + ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", "tensor",
20 + QuantizationType.INT, 2560),
21 + ("nm-testing/tinyllama-oneshot-w8-channel-a8-tensor", "channel",
22 + QuantizationType.INT, 2560),
23 ])
24 def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
21 - model_path, strategy = model_args
25 + model_path, strategy, quant_type, shape_0 = model_args
26 with vllm_runner(model_path, enforce_eager=True) as llm:
27 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
28 layer = model.model.layers[0]
@@ -34,17 +38,23 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
38 CompressedTensorsLinearMethod)
39 assert isinstance(down_proj.quant_method,
40 CompressedTensorsLinearMethod)
37 -
41 assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
42
43 assert qkv_proj.scheme.strategy == strategy
41 - assert qkv_proj.weight.dtype is torch.int8
42 - assert o_proj.weight.dtype is torch.int8
43 - assert gate_up_proj.weight.dtype is torch.int8
44 + expected_type = (torch.int8 if quant_type == QuantizationType.INT else
45 + torch.float8_e4m3fn)
46 +
47 + assert qkv_proj.weight.dtype is expected_type
48 + assert o_proj.weight.dtype is expected_type
49 + assert gate_up_proj.weight.dtype is expected_type
50
51 if qkv_proj.scheme.strategy == "tensor":
46 - assert qkv_proj.weight_scale.shard_splitter is not None
47 - assert qkv_proj.weight_scale.logical_widths is not None
52 + # Make sure it is a channelwise buffer
53 + # After running process_weights_after_loading
54 + assert len(qkv_proj.weight_scale.shape) == 2
55 + assert qkv_proj.weight_scale.shape[0] == shape_0
56 + assert qkv_proj.weight_scale.shape[1] == 1
57 + assert qkv_proj.weight_scale.dtype is torch.float32
58 assert qkv_proj.input_scale.dtype is torch.float32
59
60
tests/quantization/test_fp8.py
+17
@@ -9,6 +9,23 @@ from tests.quantization.utils import is_quant_method_supported
9 from vllm._custom_ops import scaled_fp8_quant
10 from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
11
12 +MODELS = [
13 + "neuralmagic/Meta-Llama-3-8B-Instruct-FP8",
14 + "nm-testing/Phi-3-mini-128k-instruct-FP8",
15 +]
16 +
17 +
18 +@pytest.mark.skipif(not is_quant_method_supported("fp8"),
19 + reason="FP8 is not supported on this GPU type.")
20 +@pytest.mark.parametrize("model", MODELS)
21 +def test_model_load_and_run(vllm_runner, model: str):
22 + with vllm_runner(model) as llm:
23 + # note: this does not test accuracy, just that we can run through
24 + # see lm-eval tests for accuracy
25 + outputs = llm.generate_greedy(prompts=["Hello my name is"],
26 + max_tokens=10)
27 + print(outputs[0][1])
28 +
29
30 @pytest.mark.skipif(not is_quant_method_supported("fp8"),
31 reason="FP8 is not supported on this GPU type.")
vllm/model_executor/layers/linear.py
+39 -71
@@ -41,6 +41,29 @@ def adjust_bitsandbytes_shard(param: Parameter,
41 return quantized_size, quantized_offset
42
43
44 +def adjust_scalar_to_fused_array(param, loaded_weight, shard_id):
45 + """For fused modules (QKV and MLP) we have an array of length
46 + N that holds 1 scale for each "logical" matrix. So the param
47 + is an array of length N. The loaded_weight corresponds to
48 + one of the shards on disk. Here, we slice the param based on
49 + the shard_id for loading.
50 + """
51 + qkv_idxs = {"q": 0, "k": 1, "v": 2}
52 +
53 + if isinstance(shard_id, str):
54 + shard_id = qkv_idxs[shard_id]
55 + elif not isinstance(shard_id, int):
56 + raise ValueError(f"Unknown Shard Id {shard_id}")
57 +
58 + # AutoFP8 scales do not have a shape
59 + # compressed-tensors scales do have a shape
60 + if len(loaded_weight.shape) != 0:
61 + assert loaded_weight.shape[0] == 1
62 + loaded_weight = loaded_weight[0]
63 +
64 + return param[shard_id], loaded_weight
65 +
66 +
67 class LinearMethodBase(QuantizeMethodBase):
68 """Base class for different (maybe quantized) linear methods."""
69
@@ -358,37 +381,15 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
381 output_dim = getattr(param, "output_dim", None)
382 # Special case for AQLM codebooks.
383 is_metadata = getattr(param, "is_metadata", False)
361 -
362 - param_shard_splitter = getattr(param, "shard_splitter", None)
363 -
364 - if output_dim is not None and param_shard_splitter is not None:
365 - raise NotImplementedError(
366 - "We do not currently support output_dim != None and "
367 - "shard_splitter != None for a parameter. Please open an issue."
368 - )
369 - # If a parameter has defined a shard_splitter to be used for
370 - # the weight, it should be applied before the weight is
371 - # loaded/copied to the parameter. The shard_splitter applies
372 - # logic by using the loaded_shard_id to ensure that the loaded
373 - # param is loaded to the correct location
374 - # within the parameter defined by the linear method.
375 - if loaded_shard_id is None and param_shard_splitter is not None:
376 - raise NotImplementedError(
377 - "We do not currently support loaded_shard_id == None and "
378 - "shard_splitter != None for a parameter. Please open an issue."
379 - )
380 -
381 - # Special case for Fp8 scales.
382 - fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
383 - None)
384 + # Special case for per-tensor scale to load scalar into fused array.
385 + needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False)
386
387 if loaded_shard_id is None:
388 # Loaded weight is already fused on disk (qkv/mlp).
389 if output_dim is None:
388 - # If fp8 + scale, need to send to each shard.
389 - if fp8_scales_shard_indexer is not None:
390 - param_data, loaded_weight = fp8_scales_shard_indexer(
391 - param_data, loaded_weight, loaded_shard_id)
390 + if needs_scalar_to_array is not None:
391 + param_data, loaded_weight = adjust_scalar_to_fused_array(
392 + param_data, loaded_weight, 0)
393
394 assert param_data.shape == loaded_weight.shape
395 param_data.copy_(loaded_weight)
@@ -450,15 +451,9 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
451 shard_offset = loaded_shard_id * shard_size
452 param_data = param_data.narrow(0, shard_offset, shard_size)
453
453 - # If a param_shard_splitter is defined by the LinearMethod, use it.
454 - elif param_shard_splitter is not None:
455 - logical_widths = getattr(param, "logical_widths", None)
456 - param_data, loaded_weight = param_shard_splitter(
457 - param_data, loaded_weight, loaded_shard_id, logical_widths)
458 -
459 - # Special case for Fp8 scales.
460 - elif fp8_scales_shard_indexer is not None:
461 - param_data, loaded_weight = fp8_scales_shard_indexer(
454 + # Special case for per-tensor scales in fused case.
455 + elif needs_scalar_to_array:
456 + param_data, loaded_weight = adjust_scalar_to_fused_array(
457 param_data, loaded_weight, loaded_shard_id)
458
459 else:
@@ -548,36 +543,15 @@ class QKVParallelLinear(ColumnParallelLinear):
543 # Special case for AQLM codebooks.
544 is_metadata = getattr(param, "is_metadata", False)
545
551 - param_shard_splitter = getattr(param, "shard_splitter", None)
552 -
553 - if output_dim is not None and param_shard_splitter is not None:
554 - raise NotImplementedError(
555 - "We do not currently support output_dim != None and "
556 - "shard_splitter != None for a parameter. Please open an issue."
557 - )
558 - # If a parameter has defined a shard_splitter to be used for
559 - # the weight, it should be applied before the weight is
560 - # loaded/copied to the parameter. The shard_splitter applies
561 - # logic by using the loaded_shard_id to ensure that the loaded
562 - # param is loaded to the correct location
563 - # within the parameter defined by the linear method.
564 - if loaded_shard_id is None and param_shard_splitter is not None:
565 - raise NotImplementedError(
566 - "We do not currently support loaded_shard_id == None and "
567 - "shard_splitter != None for a parameter. Please open an issue."
568 - )
569 -
570 - # Special case for Fp8 scales.
571 - fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
572 - None)
546 + # Special case for per-tensor scales in fused case.
547 + needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False)
548
549 if loaded_shard_id is None:
550 # Loaded weight is already fused on disk (qkv/mlp).
551 if output_dim is None:
577 - # If fp8 + scale, need to send to each shard.
578 - if fp8_scales_shard_indexer is not None:
579 - param_data, loaded_weight = fp8_scales_shard_indexer(
580 - param_data, loaded_weight, loaded_shard_id)
552 + if needs_scalar_to_array is not None:
553 + param_data, loaded_weight = adjust_scalar_to_fused_array(
554 + param_data, loaded_weight, 0)
555
556 assert param_data.shape == loaded_weight.shape
557 param_data.copy_(loaded_weight)
@@ -667,15 +641,9 @@ class QKVParallelLinear(ColumnParallelLinear):
641 shard_index = ["q", "k", "v"].index(loaded_shard_id)
642 param_data = param_data.narrow(0, shard_index * shard_size,
643 shard_size)
670 - # If a param_shard_splitter is defined by the LinearMethod, use it.
671 - elif param_shard_splitter is not None:
672 - logical_widths = getattr(param, "logical_widths", None)
673 - param_data, loaded_weight = param_shard_splitter(
674 - param_data, loaded_weight, loaded_shard_id, logical_widths)
675 -
676 - # Special case for Fp8 scales.
677 - elif fp8_scales_shard_indexer is not None:
678 - param_data, loaded_weight = fp8_scales_shard_indexer(
644 + # Special case for per-tensor scales in fused case.
645 + elif needs_scalar_to_array:
646 + param_data, loaded_weight = adjust_scalar_to_fused_array(
647 param_data, loaded_weight, loaded_shard_id)
648 else:
649 ignore_warning = getattr(param, "ignore_warning", False)
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+3
@@ -186,6 +186,9 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
186 def __init__(self, quantization_config: CompressedTensorsConfig):
187 self.quantization_config = quantization_config
188
189 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
190 + return layer.scheme.process_weights_after_loading(layer)
191 +
192 def create_weights(self, layer: torch.nn.Module,
193 input_size_per_partition: int,
194 output_partition_sizes: List[int], input_size: int,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py
+8
@@ -31,3 +31,11 @@ class CompressedTensorsScheme(ABC):
31
32 """
33 raise NotImplementedError
34 +
35 + @abstractmethod
36 + def process_weights_after_loading(self, layer: torch.nn.Module):
37 + """
38 + Called after weight loading is complete for any cleanup that
39 + needs to occur.
40 + """
41 + raise NotImplementedError
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
+3
@@ -18,6 +18,9 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
18 in a linear transformation.
19 """
20
21 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
22 + pass
23 +
24 def create_weights(self, layer: torch.nn.Module,
25 output_partition_sizes: List[int],
26 input_size_per_partition: int,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+3
@@ -29,6 +29,9 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
29 raise ValueError(
30 "group_size must be given when using strategy group")
31
32 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
33 + pass
34 +
35 def create_weights(self, layer: torch.nn.Module, input_size: int,
36 output_partition_sizes: List[int],
37 input_size_per_partition: int,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8.py
+42 -49
@@ -15,70 +15,63 @@ class CompressedTensorsW8A8(CompressedTensorsScheme):
15 def __init__(self, strategy: str):
16 self.strategy = strategy
17
18 - def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
19 - if isinstance(shard_id, int):
20 - return shard_id
21 -
22 - assert isinstance(shard_id, str)
23 - qkv_idxs = {"q": 0, "k": 1, "v": 2}
24 - assert shard_id in qkv_idxs
25 - return qkv_idxs[shard_id]
26 -
27 - def scales_shard_splitter(
28 - self, param: torch.Tensor, loaded_weight: torch.Tensor,
29 - shard_id: Union[str, int],
30 - logical_widths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
31 - shard_id = self._shard_id_as_int(shard_id)
32 - offset = sum(logical_widths[:shard_id])
33 - size = logical_widths[shard_id]
34 - # update loaded weight with copies for broadcast.
35 - loaded_weight = loaded_weight.repeat(size)
36 - return param[offset:offset + size], loaded_weight
18 + # Cutlass kernels support only per-tensor and per-channel cases.
19 + # So if we have a fused module (QKV, MLP) with per tensor scales (thus N
20 + # scales being passed to the kernel), we convert to the per-channel case.
21 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
22 + if (self.strategy == QuantizationStrategy.TENSOR
23 + and len(self.logical_widths) > 1):
24 +
25 + # Load the N per-tensor scales into the channelwise buffer.
26 + weight_scale_channel = torch.empty(
27 + (sum(self.logical_widths), 1),
28 + dtype=torch.float32,
29 + device=layer.weight_scale.device)
30 + start = 0
31 + for idx, logical_width in enumerate(self.logical_widths):
32 + end = start + logical_width
33 + weight_scale_channel[start:end, :] = layer.weight_scale[idx]
34 + start = end
35 +
36 + layer.weight_scale = Parameter(weight_scale_channel,
37 + requires_grad=False)
38
39 def create_weights(self, layer: torch.nn.Module,
40 output_partition_sizes: List[int],
41 input_size_per_partition: int,
42 params_dtype: torch.dtype, weight_loader: Callable,
43 **kwargs):
44 + self.logical_widths = output_partition_sizes
45
44 - is_tensor_partitioned = len(output_partition_sizes) != 1
45 - weight_scale_dim = sum(output_partition_sizes) if (
46 - is_tensor_partitioned
47 - or self.strategy == QuantizationStrategy.CHANNEL) else 1
48 -
49 - shape: Union[Tuple[int], Tuple[int, int]] = (weight_scale_dim, )
46 + # WEIGHT SCALE
47 + shape: Union[Tuple[int], Tuple[int, int]]
48 if self.strategy == QuantizationStrategy.CHANNEL:
51 - shape = (weight_scale_dim, 1)
49 + shape = (sum(self.logical_widths), 1)
50 + else:
51 + shape = (len(self.logical_widths), )
52
53 weight_scale = Parameter(torch.empty(*shape, dtype=torch.float32),
54 requires_grad=False)
55 -
55 layer.register_parameter("weight_scale", weight_scale)
57 - set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
56 + if self.strategy == QuantizationStrategy.CHANNEL:
57 + set_weight_attrs(weight_scale, {
58 + "weight_loader": weight_loader,
59 + "output_dim": 0,
60 + })
61 + else:
62 + set_weight_attrs(weight_scale, {
63 + "weight_loader": weight_loader,
64 + "needs_scalar_to_array": True,
65 + })
66
67 + # WEIGHT
68 weight = Parameter(torch.empty(sum(output_partition_sizes),
69 input_size_per_partition,
70 dtype=torch.int8),
71 requires_grad=False)
63 -
72 layer.register_parameter("weight", weight)
65 - set_weight_attrs(
66 - weight, {
67 - "input_dim": 1,
68 - "output_dim": 0,
69 - "weight_loader": weight_loader,
70 - "logical_widths": output_partition_sizes
71 - })
72 -
73 - # Don't need a shard_splitter for channel-wise quantization
74 - # Use the default loading method
75 - if self.strategy == QuantizationStrategy.CHANNEL:
76 - set_weight_attrs(weight_scale, {
77 - "output_dim": 0,
78 - })
79 - else:
80 - set_weight_attrs(
81 - weight_scale, {
82 - "logical_widths": output_partition_sizes,
83 - "shard_splitter": self.scales_shard_splitter,
84 - })
73 + set_weight_attrs(weight, {
74 + "input_dim": 1,
75 + "output_dim": 0,
76 + "weight_loader": weight_loader,
77 + })
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+3
@@ -29,6 +29,9 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
29 raise ValueError(
30 "group_size must be given when using strategy group")
31
32 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
33 + pass
34 +
35 def create_weights(self, layer: torch.nn.Module, input_size: int,
36 output_partition_sizes: List[int],
37 input_size_per_partition: int,
vllm/model_executor/layers/quantization/fp8.py
+16 -29
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional, Tuple, Union
1 +from typing import Any, Dict, List, Optional, Union
2
3 import torch
4 from torch.nn import Module
@@ -98,7 +98,6 @@ class Fp8LinearMethod(LinearMethodBase):
98 """
99
100 def __init__(self, quant_config: Fp8Config):
101 - self.fused_module_in_checkpoint = False
101 self.quant_config = quant_config
102 self.cutlass_fp8_supported = cutlass_fp8_supported()
103
@@ -114,12 +113,10 @@ class Fp8LinearMethod(LinearMethodBase):
113 requires_grad=False)
114 scale[:] = torch.finfo(torch.float8_e4m3fn).min
115 layer.register_parameter(scale_name, scale)
117 - set_weight_attrs(
118 - scale, {
119 - **extra_weight_attrs,
120 - "fp8_scales_shard_indexer":
121 - self.scales_shard_indexer,
122 - })
116 + set_weight_attrs(scale, {
117 + **extra_weight_attrs,
118 + "needs_scalar_to_array": True,
119 + })
120
121 def create_weights(
122 self,
@@ -170,26 +167,6 @@ class Fp8LinearMethod(LinearMethodBase):
167 output_partition_sizes=output_partition_sizes,
168 **extra_weight_attrs)
169
173 - def scales_shard_indexer(
174 - self, param: torch.Tensor, loaded_weight: torch.Tensor,
175 - shard_id: Optional[Union[str,
176 - int]]) -> Tuple[torch.Tensor, torch.Tensor]:
177 - qkv_idxs = {"q": 0, "k": 1, "v": 2}
178 -
179 - if shard_id is None:
180 - shard_id = 0
181 - self.fused_module_in_checkpoint = True
182 - elif isinstance(shard_id, int):
183 - pass
184 - elif isinstance(shard_id, str):
185 - if shard_id not in qkv_idxs:
186 - raise ValueError(f"Unknown shard_id: {shard_id}")
187 - shard_id = qkv_idxs[shard_id]
188 - else:
189 - ValueError(f"Shard id must be int or str but got {type(shard_id)}")
190 -
191 - return param[shard_id], loaded_weight
192 -
170 def process_weights_after_loading(self, layer: Module) -> None:
171 if (not hasattr(layer, "process_after_load")
172 or not layer.process_after_load):
@@ -212,7 +189,17 @@ class Fp8LinearMethod(LinearMethodBase):
189 # Loop over logical weights, requantizing with single scale.
190 max_w_scale = layer.weight_scale.max()
191
215 - if not self.fused_module_in_checkpoint:
192 + # QKV / MLP is fused in the on disk checkpoint if any of the
193 + # weight scales are still set to the default since we initialize
194 + # N weight scales for N shards but we only load 1 weight scale
195 + # from disk in this case. As a result, we skip dequant -> requant
196 + # since we already have quantized QKV together.
197 + # Sample Model with fused checkpoint:
198 + # * nm-testing/Phi-3-mini-128k-instruct-FP8
199 + unfused_module_in_checkpoint = (
200 + layer.weight_scale[-1] > torch.finfo(torch.float8_e4m3fn).min)
201 +
202 + if unfused_module_in_checkpoint:
203 start = 0
204 for idx, logical_width in enumerate(layer.logical_widths):
205 end = start + logical_width