A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] Support Fp8 Checkpoints (Dynamic + Static) (#4332)
Co-authored-by: Philipp Moritz <pcmoritz@gmail.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
Robert Shaw committed
Apr 30, 2024 at 17:46 UTC
111815d482ba2b724541994da12736615101ef5e
3 files changed
+307
-40
tests/models/test_fp8.py
new
+90
@@ -0,0 +1,90 @@
1
+# flake8: noqa
2
+"""Tests fp8 models against ground truth generation
3
+Note: these tests will only pass on L4 GPU.
4
+"""
5
+import os
6
+
7
+import pytest
8
+import torch
9
+from transformers import AutoTokenizer
10
+
11
+from vllm import LLM, SamplingParams
12
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
+
14
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
15
+
16
+MAX_MODEL_LEN = 1024
17
+
18
+MODELS = [
19
+ "nm-testing/Meta-Llama-3-8B-Instruct-FP8",
20
+ "meta-llama/Meta-Llama-3-8B-Instruct",
21
+]
22
+
23
+EXPECTED_STRS_MAP = {
24
+ "nm-testing/Meta-Llama-3-8B-Instruct-FP8": [
25
+ 'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
26
+ 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
27
+ 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
28
+ 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
29
+ 'Zeta-5, a highly advanced robot designed for menial labor, whirred and beep',
30
+ 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. Here',
31
+ 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
32
+ 'Here are the translations:\n\n**Japanese:** (Haya tori, nemuri nemuri)\n\n**'
33
+ ],
34
+ "meta-llama/Meta-Llama-3-8B-Instruct": [
35
+ 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
36
+ 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
37
+ 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
38
+ 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
39
+ 'In the year 2154, the robotics lab at NeuroSpark Industries was on the cusp of',
40
+ 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
41
+ 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
42
+ 'Here are the translations:\n\n**Japanese:** (Haya aki wa mushi o tsukamu'
43
+ ],
44
+}
45
+
46
+capability = torch.cuda.get_device_capability()
47
+capability = capability[0] * 10 + capability[1]
48
+fp8_not_supported = (capability <
49
+ QUANTIZATION_METHODS["fp8"].get_min_capability())
50
+
51
+
52
+@pytest.mark.skipif(fp8_not_supported,
53
+ reason="fp8 is not supported on this GPU type.")
54
+@pytest.mark.parametrize("model_name", MODELS)
55
+def test_models(
56
+ example_prompts,
57
+ model_name,
58
+) -> None:
59
+ model = LLM(model=model_name,
60
+ max_model_len=MAX_MODEL_LEN,
61
+ enforce_eager=True,
62
+ quantization="fp8")
63
+
64
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
65
+ formatted_prompts = [
66
+ tokenizer.apply_chat_template([{
67
+ "role": "user",
68
+ "content": prompt
69
+ }],
70
+ tokenize=False,
71
+ add_generation_prompt=True)
72
+ for prompt in example_prompts
73
+ ]
74
+
75
+ params = SamplingParams(max_tokens=20, temperature=0)
76
+ generations = []
77
+ # Note: these need to be run 1 at a time due to numerical precision,
78
+ # since the expected strs were generated this way.
79
+ for prompt in formatted_prompts:
80
+ outputs = model.generate(prompt, params)
81
+ generations.append(outputs[0].outputs[0].text)
82
+ del model
83
+
84
+ print(generations)
85
+ expected_strs = EXPECTED_STRS_MAP[model_name]
86
+ for i in range(len(example_prompts)):
87
+ generated_str = generations[i]
88
+ expected_str = expected_strs[i]
89
+ assert expected_str == generated_str, (
90
+ f"Test{i}:\nExpected: {expected_str!r}\nvLLM: {generated_str!r}")
vllm/model_executor/layers/linear.py
+48
-10
@@ -246,6 +246,10 @@ class ColumnParallelLinear(LinearBase):
246
self.register_parameter("bias", None)
247
248
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
249
+ # Special case for Fp8 scales.
250
+ fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
251
+ None)
252
+
253
tp_rank = get_tensor_model_parallel_rank()
254
output_dim = getattr(param, "output_dim", None)
255
param_data = param.data
@@ -254,6 +258,12 @@ class ColumnParallelLinear(LinearBase):
258
start_idx = tp_rank * shard_size
259
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
260
shard_size)
261
+ # Special case for Fp8 scales.
262
+ elif fp8_scales_shard_indexer is not None:
263
+ param_data, loaded_weight = fp8_scales_shard_indexer(param_data,
264
+ loaded_weight,
265
+ shard_id=0)
266
+
267
assert param_data.shape == loaded_weight.shape
268
param_data.copy_(loaded_weight)
269
@@ -317,7 +327,12 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
327
328
param_data = param.data
329
output_dim = getattr(param, "output_dim", None)
330
+ # Special case for AQLM codebooks.
331
is_metadata = getattr(param, "is_metadata", False)
332
+ # Special case for Fp8 scales.
333
+ fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
334
+ None)
335
+
336
if loaded_shard_id is None:
337
# Loaded weight is already packed.
338
if output_dim is None:
@@ -331,14 +346,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
346
current_shard_offset += output_size
347
packed_dim = getattr(param, "packed_dim", None)
348
for shard_id, shard_offset, shard_size in shard_offsets:
349
+ # Special case for Quantization.
350
# If quantized, we need to adjust the offset and size to account
351
# for the packing.
352
if packed_dim == output_dim:
353
shard_size = shard_size // param.pack_factor
354
shard_offset = shard_offset // param.pack_factor
339
-
340
- # If marlin, we need to adjust the offset and size to
341
- # account for the tiling.
355
+ # Special case for Marlin.
356
shard_size, shard_offset = adjust_marlin_shard(
357
param, shard_size, shard_offset)
358
@@ -353,15 +367,14 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
367
if output_dim is not None:
368
shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
369
shard_size = self.output_sizes[loaded_shard_id] // tp_size
370
+ # Special case for quantization.
371
# If quantized, we need to adjust the offset and size to account
372
# for the packing.
373
packed_dim = getattr(param, "packed_dim", None)
374
if packed_dim == output_dim:
375
shard_size = shard_size // param.pack_factor
376
shard_offset = shard_offset // param.pack_factor
362
-
363
- # If marlin, we need to adjust the offset and size to
364
- # account for the tiling.
377
+ # Special case for Marlin.
378
shard_size, shard_offset = adjust_marlin_shard(
379
param, shard_size, shard_offset)
380
@@ -370,11 +383,17 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
383
start_idx = tp_rank * shard_size
384
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
385
shard_size)
386
+ # Special case for AQLM codebooks.
387
elif is_metadata:
388
# metadata indicates fixed size concatenated along dim 0
389
shard_size = loaded_weight.shape[0]
390
shard_offset = loaded_shard_id * shard_size
391
param_data = param_data.narrow(0, shard_offset, shard_size)
392
+ # Special case for Fp8 scales.
393
+ elif fp8_scales_shard_indexer is not None:
394
+ param_data, loaded_weight = fp8_scales_shard_indexer(
395
+ param_data, loaded_weight, loaded_shard_id)
396
+
397
else:
398
ignore_warning = getattr(param, "ignore_warning", False)
399
if not ignore_warning:
@@ -455,7 +474,11 @@ class QKVParallelLinear(ColumnParallelLinear):
474
loaded_shard_id: Optional[str] = None):
475
param_data = param.data
476
output_dim = getattr(param, "output_dim", None)
477
+ # Special case for AQLM codebooks.
478
is_metadata = getattr(param, "is_metadata", False)
479
+ # Special case for Fp8 scales.
480
+ fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
481
+ None)
482
483
if loaded_shard_id is None:
484
# Loaded weight is already packed.
@@ -473,14 +496,14 @@ class QKVParallelLinear(ColumnParallelLinear):
496
]
497
packed_dim = getattr(param, "packed_dim", None)
498
for shard_id, shard_offset, shard_size in shard_offsets:
499
+ # Special case for Quantized Weights.
500
# If quantized, we need to adjust the offset and size to account
501
# for the packing.
502
if packed_dim == output_dim:
503
shard_size = shard_size // param.pack_factor
504
shard_offset = shard_offset // param.pack_factor
505
482
- # If marlin, we need to adjust the offset and size to
483
- # account for the tiling.
506
+ # Special case for Marlin.
507
shard_size, shard_offset = adjust_marlin_shard(
508
param, shard_size, shard_offset)
509
@@ -502,6 +525,7 @@ class QKVParallelLinear(ColumnParallelLinear):
525
shard_offset = (self.num_heads +
526
self.num_kv_heads) * self.head_size
527
shard_size = self.num_kv_heads * self.head_size
528
+ # Special case for Quantized Weights.
529
# If quantized, we need to adjust the offset and size to account
530
# for the packing.
531
packed_dim = getattr(param, "packed_dim", None)
@@ -509,8 +533,7 @@ class QKVParallelLinear(ColumnParallelLinear):
533
shard_size = shard_size // param.pack_factor
534
shard_offset = shard_offset // param.pack_factor
535
512
- # If marlin, we need to adjust the offset and size to
513
- # account for the tiling.
536
+ # Special case for Marlin.
537
shard_size, shard_offset = adjust_marlin_shard(
538
param, shard_size, shard_offset)
539
@@ -523,12 +546,17 @@ class QKVParallelLinear(ColumnParallelLinear):
546
start_idx = shard_id * shard_size
547
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
548
shard_size)
549
+ # Special case for for AQLM codebooks.
550
elif is_metadata:
551
# metadata indicates fixed size concatenated along dim 0
552
shard_size = loaded_weight.shape[0]
553
shard_index = ["q", "k", "v"].index(loaded_shard_id)
554
param_data = param_data.narrow(0, shard_index * shard_size,
555
shard_size)
556
+ # Special case for Fp8 scales.
557
+ elif fp8_scales_shard_indexer is not None:
558
+ param_data, loaded_weight = fp8_scales_shard_indexer(
559
+ param_data, loaded_weight, loaded_shard_id)
560
else:
561
ignore_warning = getattr(param, "ignore_warning", False)
562
if not ignore_warning:
@@ -611,6 +639,10 @@ class RowParallelLinear(LinearBase):
639
self.register_parameter("bias", None)
640
641
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
642
+ # Special case for Fp8 scales.
643
+ fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
644
+ None)
645
+
646
tp_rank = get_tensor_model_parallel_rank()
647
input_dim = getattr(param, "input_dim", None)
648
param_data = param.data
@@ -619,6 +651,12 @@ class RowParallelLinear(LinearBase):
651
start_idx = tp_rank * shard_size
652
loaded_weight = loaded_weight.narrow(input_dim, start_idx,
653
shard_size)
654
+ # Special case for Fp8 scales.
655
+ elif fp8_scales_shard_indexer is not None:
656
+ param_data, loaded_weight = fp8_scales_shard_indexer(param_data,
657
+ loaded_weight,
658
+ shard_id=0)
659
+
660
assert param_data.shape == loaded_weight.shape
661
param_data.copy_(loaded_weight)
662
vllm/model_executor/layers/quantization/fp8.py
+169
-30
@@ -1,23 +1,36 @@
1
-from typing import Any, Dict, List, Optional
1
+from typing import Any, Dict, List, Optional, Tuple, Union
2
3
import torch
4
from torch.nn import Module
5
from torch.nn.parameter import Parameter
6
7
from vllm import _custom_ops as ops
8
+from vllm.logger import init_logger
9
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10
from vllm.model_executor.layers.quantization.base_config import (
10
- QuantizationConfig, QuantizeMethodBase)
11
+ QuantizationConfig)
12
from vllm.model_executor.utils import set_weight_attrs
13
14
+ACTIVATION_SCHEMES = ["static", "dynamic"]
15
+
16
+logger = init_logger(__name__)
17
+
18
19
class Fp8Config(QuantizationConfig):
20
"""Config class for FP8."""
21
22
def __init__(
23
self,
24
+ is_checkpoint_fp8_serialized: bool = False,
25
activation_scheme: str = "dynamic",
26
) -> None:
27
+ self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized
28
+ if is_checkpoint_fp8_serialized:
29
+ logger.warning("Detected fp8 checkpoint. Please note that the "
30
+ "format is experimental and subject to change.")
31
+ if activation_scheme not in ACTIVATION_SCHEMES:
32
+ raise ValueError(
33
+ f"Unsupported activation scheme {activation_scheme}")
34
self.activation_scheme = activation_scheme
35
36
@classmethod
@@ -30,10 +43,7 @@ class Fp8Config(QuantizationConfig):
43
44
@classmethod
45
def get_min_capability(cls) -> int:
33
- # TODO: PyTorch 2.3.0+ is required to run FP8 on
34
- # SM 89 (e.g. Ada) GPUs. Specifically, this PR has to
35
- # be included: https://github.com/pytorch/pytorch/pull/118881
36
- return 90
46
+ return 89
47
48
@classmethod
49
def get_config_filenames(cls) -> List[str]:
@@ -41,11 +51,14 @@ class Fp8Config(QuantizationConfig):
51
52
@classmethod
53
def from_config(cls, config: Dict[str, Any]) -> "Fp8Config":
54
+ quant_method = cls.get_from_keys(config, ["quant_method"])
55
+ is_checkpoint_fp8_serialized = ("fp8" in quant_method)
56
activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
45
- return cls(activation_scheme)
57
+ return cls(is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
58
+ activation_scheme=activation_scheme)
59
60
def get_quant_method(
48
- self, layer: torch.nn.Module) -> Optional["QuantizeMethodBase"]:
61
+ self, layer: torch.nn.Module) -> Optional["Fp8LinearMethod"]:
62
if isinstance(layer, LinearBase):
63
return Fp8LinearMethod(self)
64
return None
@@ -56,8 +69,12 @@ class Fp8Config(QuantizationConfig):
69
70
class Fp8LinearMethod(LinearMethodBase):
71
"""Linear method for FP8.
59
- We now support common FP16/BF16 model checkpoints ONLY. The weight
60
- scaling factor will be initialized after the model weights are loaded.
72
+ Supports loading FP8 checkpoints with static weight scale and
73
+ dynamic/static activation scale.
74
+
75
+ Also supports loading quantized FP16/BF16 model checkpoints with dynamic
76
+ activation scaling. The weight scaling factor will be initialized after
77
+ the model weights are loaded.
78
79
Limitations:
80
1. Only support per-tensor quantization due to torch._scaled_mm support.
@@ -71,6 +88,24 @@ class Fp8LinearMethod(LinearMethodBase):
88
def __init__(self, quant_config: Fp8Config):
89
self.quant_config = quant_config
90
91
+ def _create_scale_param(
92
+ self,
93
+ scale_name: str,
94
+ layer: torch.nn.Module,
95
+ output_partition_sizes: List[int],
96
+ **extra_weight_attrs,
97
+ ) -> None:
98
+ scale = Parameter(torch.empty(len(output_partition_sizes),
99
+ dtype=torch.float32),
100
+ requires_grad=False)
101
+ layer.register_parameter(scale_name, scale)
102
+ set_weight_attrs(
103
+ scale, {
104
+ **extra_weight_attrs,
105
+ "fp8_scales_shard_indexer":
106
+ self.scales_shard_indexer,
107
+ })
108
+
109
def create_weights(
110
self,
111
layer: torch.nn.Module,
@@ -81,46 +116,150 @@ class Fp8LinearMethod(LinearMethodBase):
116
params_dtype: torch.dtype,
117
**extra_weight_attrs,
118
):
119
+ del input_size, output_size
120
output_size_per_partition = sum(output_partition_sizes)
121
+
122
+ layer.process_after_load = True
123
+ layer.logical_widths = output_partition_sizes
124
+
125
+ # WEIGHT
126
+ weight_dtype = (torch.float8_e4m3fn
127
+ if self.quant_config.is_checkpoint_fp8_serialized else
128
+ params_dtype)
129
weight = Parameter(torch.empty(output_size_per_partition,
130
input_size_per_partition,
87
- dtype=params_dtype),
131
+ dtype=weight_dtype),
132
requires_grad=False)
133
layer.register_parameter("weight", weight)
90
- set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
91
- set_weight_attrs(weight, extra_weight_attrs)
134
+ set_weight_attrs(weight, {
135
+ **extra_weight_attrs,
136
+ "input_dim": 1,
137
+ "output_dim": 0,
138
+ })
139
93
- w_scale = Parameter(
94
- torch.empty(1, dtype=torch.float32),
95
- requires_grad=False,
96
- )
97
- layer.register_parameter("weight_scaling_factor", w_scale)
140
+ # If checkpoint is serialized fp8, load them.
141
+ # Otherwise, wait until process_weights_after_loading.
142
+ if self.quant_config.is_checkpoint_fp8_serialized:
143
+ # WEIGHT SCALE
144
+ self._create_scale_param(
145
+ scale_name="weight_scale",
146
+ layer=layer,
147
+ output_partition_sizes=output_partition_sizes,
148
+ **extra_weight_attrs)
149
+
150
+ # ACTIVATION SCALE
151
+ if self.quant_config.activation_scheme == "static":
152
+ self._create_scale_param(
153
+ scale_name="act_scale",
154
+ layer=layer,
155
+ output_partition_sizes=output_partition_sizes,
156
+ **extra_weight_attrs)
157
+
158
+ def scales_shard_indexer(
159
+ self, param: torch.Tensor, loaded_weight: torch.Tensor,
160
+ shard_id: Union[str, int]) -> Tuple[torch.Tensor, torch.Tensor]:
161
+ qkv_idxs = {"q": 0, "k": 1, "v": 2}
162
+
163
+ if isinstance(shard_id, int):
164
+ pass
165
+ elif isinstance(shard_id, str):
166
+ if shard_id not in qkv_idxs:
167
+ raise ValueError(f"Unknown shard_id: {shard_id}")
168
+ shard_id = qkv_idxs[shard_id]
169
+ else:
170
+ ValueError(f"Shard id must be int or str but got {type(shard_id)}")
171
+
172
+ return param[shard_id], loaded_weight
173
174
def process_weights_after_loading(self, layer: Module) -> None:
100
- # Although the quant_method is propagated to all layers,
101
- # only linear layers invoke "create_weights". So we check
102
- # whether "weight_scaling_facor" is registered to determine
103
- # whether the layer is a linear layer that requires quantization.
104
- if not hasattr(layer, "weight_scaling_factor"):
175
+ if (not hasattr(layer, "process_after_load")
176
+ or not layer.process_after_load):
177
+ return
178
+
179
+ # If checkpoint is fp/bf16 (not serialized fp8), quantize the weights.
180
+ if not self.quant_config.is_checkpoint_fp8_serialized:
181
+ qweight, weight_scale = ops.scaled_fp8_quant(layer.weight,
182
+ scale=None)
183
+ layer.weight = Parameter(qweight.t(), requires_grad=False)
184
+ layer.weight_scale = Parameter(weight_scale, requires_grad=False)
185
+ layer.logical_widths = None
186
+ layer.act_scale = None
187
return
188
107
- qweight, weight_scale = ops.scaled_fp8_quant(layer.weight)
108
- # torch._scaled_mm requires column-major in the second
109
- # input (weight), so we transpose the quantized weight.
110
- layer.weight = Parameter(qweight.t(), requires_grad=False)
111
- layer.weight_scaling_factor.data.copy_(weight_scale)
189
+ # If checkpoint is fp8, requantize the separately quantized logical
190
+ # weights into a single fp8 weight with a single weight scale.
191
+ else:
192
+ # WEIGHT_SCALE / WEIGHT
193
+ # Loop over logical weights, requantizing with single scale.
194
+ max_w_scale = layer.weight_scale.max()
195
+ start = 0
196
+ for idx, logical_width in enumerate(layer.logical_widths):
197
+ end = start + logical_width
198
+ weight_dq = per_tensor_dequantize(layer.weight[start:end, :],
199
+ layer.weight_scale[idx])
200
+
201
+ layer.weight[start:end, :] = per_tensor_quantize(
202
+ weight_dq, layer.weight_scale.max())
203
+ start = end
204
+ layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
205
+
206
+ # WEIGHT
207
+ # Transpose weight for passing to torch._scaled_mm
208
+ weight = layer.weight
209
+ layer.weight = Parameter(weight.t(), requires_grad=False)
210
+
211
+ # ACT_SCALE
212
+ # Dynamic: set to None (required input to ops.scaled_fp8_quant).
213
+ # Static: set to max of the act_scales (since they are equal).
214
+ if self.quant_config.activation_scheme == "dynamic":
215
+ layer.act_scale = None
216
+ elif self.quant_config.activation_scheme == "static":
217
+ if not all_close_1d(layer.act_scale):
218
+ raise ValueError(
219
+ "All the act_scales for the logical weights of a layer "
220
+ f"must be equal. But got {layer.act_scale}")
221
+ layer.act_scale = Parameter(layer.act_scale.max(),
222
+ requires_grad=False)
223
+ else:
224
+ raise ValueError(
225
+ f"Unknown scheme {self.quant_config.activation_scheme}")
226
227
def apply(self,
228
layer: torch.nn.Module,
229
x: torch.Tensor,
230
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
117
- qinput, x_scale = ops.scaled_fp8_quant(x)
231
+ # ops.scaled_fp8_quant supports both dynamic and static quant.
232
+ # If dynamic, layer.act_scale is None and x_scale computed from x.
233
+ # If static, layer.act_scale is scalar and x_scale set to act_scale.
234
+ qinput, x_scale = ops.scaled_fp8_quant(x, layer.act_scale)
235
+
236
+ # Fused GEMM_DQ
237
output, _ = torch._scaled_mm(
238
qinput,
239
layer.weight,
240
out_dtype=x.dtype,
241
scale_a=x_scale,
123
- scale_b=layer.weight_scaling_factor,
242
+ scale_b=layer.weight_scale,
243
bias=bias,
244
)
245
+
246
return output
247
+
248
+
249
+def all_close_1d(x: torch.Tensor) -> bool:
250
+ assert len(x.shape) == 1
251
+ return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
252
+
253
+
254
+def per_tensor_quantize(tensor: torch.Tensor,
255
+ inv_scale: float) -> torch.Tensor:
256
+ finfo = torch.finfo(torch.float8_e4m3fn)
257
+ qweight = (tensor / inv_scale).clamp(min=finfo.min, max=finfo.max)
258
+ return qweight.to(torch.float8_e4m3fn)
259
+
260
+
261
+def per_tensor_dequantize(tensor: torch.Tensor,
262
+ inv_scale: float) -> torch.Tensor:
263
+ fake_qweight = tensor.to(torch.float16)
264
+ dq_weight = fake_qweight * inv_scale
265
+ return dq_weight