@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e16fa99a6a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Update fbgemmfp8 to use `vLLMParameters` (#7972)

Co-authored-by: Michael Goin <michael@neuralmagic.com>

Dipika Sikka committed Sep 3, 2024 at 22:12 UTC e16fa99a6ad5bae4aedfb76121d4e622d27f81c3
3 files changed +22 -41
vllm/model_executor/layers/linear.py
+1 -1
@@ -26,7 +26,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
26 "CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
27 "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
28 "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
29 - "TPUInt8LinearMethod", "GPTQLinearMethod"
29 + "TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod"
30 ]
31
32
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+21 -13
@@ -15,8 +15,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
15 from vllm.model_executor.layers.quantization.utils.quant_utils import (
16 is_layer_skipped)
17 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18 - apply_fp8_linear, create_per_channel_scale_param)
19 -from vllm.model_executor.utils import set_weight_attrs
18 + apply_fp8_linear)
19 +from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
20 + ModelWeightParameter)
21 from vllm.platforms import current_platform
22
23 logger = init_logger(__name__)
@@ -85,6 +86,7 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
86 params_dtype: torch.dtype,
87 **extra_weight_attrs,
88 ):
89 + weight_loader = extra_weight_attrs.get("weight_loader")
90 del input_size, output_size
91 output_size_per_partition = sum(output_partition_sizes)
92
@@ -95,20 +97,21 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
97 layer.orig_dtype = params_dtype
98
99 # WEIGHT
98 - weight = Parameter(torch.empty(output_size_per_partition,
99 - input_size_per_partition,
100 - dtype=torch.float8_e4m3fn),
101 - requires_grad=False)
100 + weight = ModelWeightParameter(data=torch.empty(
101 + output_size_per_partition,
102 + input_size_per_partition,
103 + dtype=torch.float8_e4m3fn),
104 + input_dim=1,
105 + output_dim=0,
106 + weight_loader=weight_loader)
107 layer.register_parameter("weight", weight)
103 - set_weight_attrs(weight, {
104 - "input_dim": 1,
105 - "output_dim": 0,
106 - **extra_weight_attrs,
107 - })
108
109 # WEIGHT SCALE
110 - weight_scale = create_per_channel_scale_param(output_partition_sizes,
111 - **extra_weight_attrs)
110 + weight_scale = ChannelQuantScaleParameter(data=torch.empty(
111 + (sum(output_partition_sizes), 1), dtype=torch.float32),
112 + output_dim=0,
113 + weight_loader=weight_loader)
114 + weight_scale[:] = torch.finfo(torch.float32).min
115 layer.register_parameter("weight_scale", weight_scale)
116
117 # INPUT SCALE UPPER BOUND
@@ -118,6 +121,11 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
121 layer.input_scale_ub = input_scale_ub
122
123 def process_weights_after_loading(self, layer: Module) -> None:
124 + # required by torch.compile
125 + layer.weight_scale = Parameter(layer.weight_scale.data,
126 + requires_grad=False)
127 + layer.weight = Parameter(layer.weight.data, requires_grad=False)
128 +
129 weight = layer.weight
130 layer.weight = Parameter(weight.t(), requires_grad=False)
131
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
-27
@@ -1,10 +1,8 @@
1 from typing import List, Optional, Tuple, Union
2
3 import torch
4 -from torch.nn import Parameter
4
5 from vllm import _custom_ops as ops
7 -from vllm.model_executor.utils import set_weight_attrs
6 from vllm.platforms import current_platform
7 from vllm.utils import is_hip
8
@@ -38,31 +36,6 @@ def all_close_1d(x: torch.Tensor) -> bool:
36 return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
37
38
41 -def create_per_tensor_scale_param(
42 - output_partition_sizes: List[int],
43 - **extra_weight_attrs,
44 -) -> Parameter:
45 - scale = Parameter(torch.empty(len(output_partition_sizes),
46 - dtype=torch.float32),
47 - requires_grad=False)
48 - scale[:] = torch.finfo(torch.float32).min
49 - set_weight_attrs(scale, {
50 - "needs_scalar_to_array": True,
51 - **extra_weight_attrs
52 - })
53 - return scale
54 -
55 -
56 -def create_per_channel_scale_param(output_partition_sizes: List[int],
57 - **extra_weight_attrs) -> Parameter:
58 - scale = Parameter(torch.empty((sum(output_partition_sizes), 1),
59 - dtype=torch.float32),
60 - requires_grad=False)
61 - scale[:] = torch.finfo(torch.float32).min
62 - set_weight_attrs(scale, {"output_dim": 0, **extra_weight_attrs})
63 - return scale
64 -
65 -
39 def convert_to_channelwise(
40 weight_scale: torch.Tensor,
41 logical_widths: List[int]) -> Tuple[torch.Tensor, torch.Tensor]: