A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Update fbgemmfp8 to use `vLLMParameters` (#7972)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Dipika Sikka committed
Sep 3, 2024 at 22:12 UTC
e16fa99a6ad5bae4aedfb76121d4e622d27f81c3
3 files changed
+22
-41
vllm/model_executor/layers/linear.py
+1
-1
@@ -26,7 +26,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
26
"CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
27
"AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
28
"MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
29
- "TPUInt8LinearMethod", "GPTQLinearMethod"
29
+ "TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod"
30
]
31
32
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+21
-13
@@ -15,8 +15,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
15
from vllm.model_executor.layers.quantization.utils.quant_utils import (
16
is_layer_skipped)
17
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18
- apply_fp8_linear, create_per_channel_scale_param)
19
-from vllm.model_executor.utils import set_weight_attrs
18
+ apply_fp8_linear)
19
+from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
20
+ ModelWeightParameter)
21
from vllm.platforms import current_platform
22
23
logger = init_logger(__name__)
@@ -85,6 +86,7 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
86
params_dtype: torch.dtype,
87
**extra_weight_attrs,
88
):
89
+ weight_loader = extra_weight_attrs.get("weight_loader")
90
del input_size, output_size
91
output_size_per_partition = sum(output_partition_sizes)
92
@@ -95,20 +97,21 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
97
layer.orig_dtype = params_dtype
98
99
# WEIGHT
98
- weight = Parameter(torch.empty(output_size_per_partition,
99
- input_size_per_partition,
100
- dtype=torch.float8_e4m3fn),
101
- requires_grad=False)
100
+ weight = ModelWeightParameter(data=torch.empty(
101
+ output_size_per_partition,
102
+ input_size_per_partition,
103
+ dtype=torch.float8_e4m3fn),
104
+ input_dim=1,
105
+ output_dim=0,
106
+ weight_loader=weight_loader)
107
layer.register_parameter("weight", weight)
103
- set_weight_attrs(weight, {
104
- "input_dim": 1,
105
- "output_dim": 0,
106
- **extra_weight_attrs,
107
- })
108
109
# WEIGHT SCALE
110
- weight_scale = create_per_channel_scale_param(output_partition_sizes,
111
- **extra_weight_attrs)
110
+ weight_scale = ChannelQuantScaleParameter(data=torch.empty(
111
+ (sum(output_partition_sizes), 1), dtype=torch.float32),
112
+ output_dim=0,
113
+ weight_loader=weight_loader)
114
+ weight_scale[:] = torch.finfo(torch.float32).min
115
layer.register_parameter("weight_scale", weight_scale)
116
117
# INPUT SCALE UPPER BOUND
@@ -118,6 +121,11 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
121
layer.input_scale_ub = input_scale_ub
122
123
def process_weights_after_loading(self, layer: Module) -> None:
124
+ # required by torch.compile
125
+ layer.weight_scale = Parameter(layer.weight_scale.data,
126
+ requires_grad=False)
127
+ layer.weight = Parameter(layer.weight.data, requires_grad=False)
128
+
129
weight = layer.weight
130
layer.weight = Parameter(weight.t(), requires_grad=False)
131
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
-27
@@ -1,10 +1,8 @@
1
from typing import List, Optional, Tuple, Union
2
3
import torch
4
-from torch.nn import Parameter
4
5
from vllm import _custom_ops as ops
7
-from vllm.model_executor.utils import set_weight_attrs
6
from vllm.platforms import current_platform
7
from vllm.utils import is_hip
8
@@ -38,31 +36,6 @@ def all_close_1d(x: torch.Tensor) -> bool:
36
return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
37
38
41
-def create_per_tensor_scale_param(
42
- output_partition_sizes: List[int],
43
- **extra_weight_attrs,
44
-) -> Parameter:
45
- scale = Parameter(torch.empty(len(output_partition_sizes),
46
- dtype=torch.float32),
47
- requires_grad=False)
48
- scale[:] = torch.finfo(torch.float32).min
49
- set_weight_attrs(scale, {
50
- "needs_scalar_to_array": True,
51
- **extra_weight_attrs
52
- })
53
- return scale
54
-
55
-
56
-def create_per_channel_scale_param(output_partition_sizes: List[int],
57
- **extra_weight_attrs) -> Parameter:
58
- scale = Parameter(torch.empty((sum(output_partition_sizes), 1),
59
- dtype=torch.float32),
60
- requires_grad=False)
61
- scale[:] = torch.finfo(torch.float32).min
62
- set_weight_attrs(scale, {"output_dim": 0, **extra_weight_attrs})
63
- return scale
64
-
65
-
39
def convert_to_channelwise(
40
weight_scale: torch.Tensor,
41
logical_widths: List[int]) -> Tuple[torch.Tensor, torch.Tensor]: