@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 111815d482
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Kernel] Support Fp8 Checkpoints (Dynamic + Static) (#4332)

Co-authored-by: Philipp Moritz <pcmoritz@gmail.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Robert Shaw committed Apr 30, 2024 at 17:46 UTC 111815d482ba2b724541994da12736615101ef5e
3 files changed +307 -40
tests/models/test_fp8.py new
+90
@@ -0,0 +1,90 @@
1 +# flake8: noqa
2 +"""Tests fp8 models against ground truth generation
3 +Note: these tests will only pass on L4 GPU.
4 +"""
5 +import os
6 +
7 +import pytest
8 +import torch
9 +from transformers import AutoTokenizer
10 +
11 +from vllm import LLM, SamplingParams
12 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13 +
14 +os.environ["TOKENIZERS_PARALLELISM"] = "true"
15 +
16 +MAX_MODEL_LEN = 1024
17 +
18 +MODELS = [
19 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8",
20 + "meta-llama/Meta-Llama-3-8B-Instruct",
21 +]
22 +
23 +EXPECTED_STRS_MAP = {
24 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8": [
25 + 'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
26 + 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
27 + 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
28 + 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
29 + 'Zeta-5, a highly advanced robot designed for menial labor, whirred and beep',
30 + 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. Here',
31 + 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
32 + 'Here are the translations:\n\n**Japanese:** (Haya tori, nemuri nemuri)\n\n**'
33 + ],
34 + "meta-llama/Meta-Llama-3-8B-Instruct": [
35 + 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
36 + 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
37 + 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
38 + 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
39 + 'In the year 2154, the robotics lab at NeuroSpark Industries was on the cusp of',
40 + 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
41 + 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
42 + 'Here are the translations:\n\n**Japanese:** (Haya aki wa mushi o tsukamu'
43 + ],
44 +}
45 +
46 +capability = torch.cuda.get_device_capability()
47 +capability = capability[0] * 10 + capability[1]
48 +fp8_not_supported = (capability <
49 + QUANTIZATION_METHODS["fp8"].get_min_capability())
50 +
51 +
52 +@pytest.mark.skipif(fp8_not_supported,
53 + reason="fp8 is not supported on this GPU type.")
54 +@pytest.mark.parametrize("model_name", MODELS)
55 +def test_models(
56 + example_prompts,
57 + model_name,
58 +) -> None:
59 + model = LLM(model=model_name,
60 + max_model_len=MAX_MODEL_LEN,
61 + enforce_eager=True,
62 + quantization="fp8")
63 +
64 + tokenizer = AutoTokenizer.from_pretrained(model_name)
65 + formatted_prompts = [
66 + tokenizer.apply_chat_template([{
67 + "role": "user",
68 + "content": prompt
69 + }],
70 + tokenize=False,
71 + add_generation_prompt=True)
72 + for prompt in example_prompts
73 + ]
74 +
75 + params = SamplingParams(max_tokens=20, temperature=0)
76 + generations = []
77 + # Note: these need to be run 1 at a time due to numerical precision,
78 + # since the expected strs were generated this way.
79 + for prompt in formatted_prompts:
80 + outputs = model.generate(prompt, params)
81 + generations.append(outputs[0].outputs[0].text)
82 + del model
83 +
84 + print(generations)
85 + expected_strs = EXPECTED_STRS_MAP[model_name]
86 + for i in range(len(example_prompts)):
87 + generated_str = generations[i]
88 + expected_str = expected_strs[i]
89 + assert expected_str == generated_str, (
90 + f"Test{i}:\nExpected: {expected_str!r}\nvLLM: {generated_str!r}")
vllm/model_executor/layers/linear.py
+48 -10
@@ -246,6 +246,10 @@ class ColumnParallelLinear(LinearBase):
246 self.register_parameter("bias", None)
247
248 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
249 + # Special case for Fp8 scales.
250 + fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
251 + None)
252 +
253 tp_rank = get_tensor_model_parallel_rank()
254 output_dim = getattr(param, "output_dim", None)
255 param_data = param.data
@@ -254,6 +258,12 @@ class ColumnParallelLinear(LinearBase):
258 start_idx = tp_rank * shard_size
259 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
260 shard_size)
261 + # Special case for Fp8 scales.
262 + elif fp8_scales_shard_indexer is not None:
263 + param_data, loaded_weight = fp8_scales_shard_indexer(param_data,
264 + loaded_weight,
265 + shard_id=0)
266 +
267 assert param_data.shape == loaded_weight.shape
268 param_data.copy_(loaded_weight)
269
@@ -317,7 +327,12 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
327
328 param_data = param.data
329 output_dim = getattr(param, "output_dim", None)
330 + # Special case for AQLM codebooks.
331 is_metadata = getattr(param, "is_metadata", False)
332 + # Special case for Fp8 scales.
333 + fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
334 + None)
335 +
336 if loaded_shard_id is None:
337 # Loaded weight is already packed.
338 if output_dim is None:
@@ -331,14 +346,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
346 current_shard_offset += output_size
347 packed_dim = getattr(param, "packed_dim", None)
348 for shard_id, shard_offset, shard_size in shard_offsets:
349 + # Special case for Quantization.
350 # If quantized, we need to adjust the offset and size to account
351 # for the packing.
352 if packed_dim == output_dim:
353 shard_size = shard_size // param.pack_factor
354 shard_offset = shard_offset // param.pack_factor
339 -
340 - # If marlin, we need to adjust the offset and size to
341 - # account for the tiling.
355 + # Special case for Marlin.
356 shard_size, shard_offset = adjust_marlin_shard(
357 param, shard_size, shard_offset)
358
@@ -353,15 +367,14 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
367 if output_dim is not None:
368 shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
369 shard_size = self.output_sizes[loaded_shard_id] // tp_size
370 + # Special case for quantization.
371 # If quantized, we need to adjust the offset and size to account
372 # for the packing.
373 packed_dim = getattr(param, "packed_dim", None)
374 if packed_dim == output_dim:
375 shard_size = shard_size // param.pack_factor
376 shard_offset = shard_offset // param.pack_factor
362 -
363 - # If marlin, we need to adjust the offset and size to
364 - # account for the tiling.
377 + # Special case for Marlin.
378 shard_size, shard_offset = adjust_marlin_shard(
379 param, shard_size, shard_offset)
380
@@ -370,11 +383,17 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
383 start_idx = tp_rank * shard_size
384 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
385 shard_size)
386 + # Special case for AQLM codebooks.
387 elif is_metadata:
388 # metadata indicates fixed size concatenated along dim 0
389 shard_size = loaded_weight.shape[0]
390 shard_offset = loaded_shard_id * shard_size
391 param_data = param_data.narrow(0, shard_offset, shard_size)
392 + # Special case for Fp8 scales.
393 + elif fp8_scales_shard_indexer is not None:
394 + param_data, loaded_weight = fp8_scales_shard_indexer(
395 + param_data, loaded_weight, loaded_shard_id)
396 +
397 else:
398 ignore_warning = getattr(param, "ignore_warning", False)
399 if not ignore_warning:
@@ -455,7 +474,11 @@ class QKVParallelLinear(ColumnParallelLinear):
474 loaded_shard_id: Optional[str] = None):
475 param_data = param.data
476 output_dim = getattr(param, "output_dim", None)
477 + # Special case for AQLM codebooks.
478 is_metadata = getattr(param, "is_metadata", False)
479 + # Special case for Fp8 scales.
480 + fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
481 + None)
482
483 if loaded_shard_id is None:
484 # Loaded weight is already packed.
@@ -473,14 +496,14 @@ class QKVParallelLinear(ColumnParallelLinear):
496 ]
497 packed_dim = getattr(param, "packed_dim", None)
498 for shard_id, shard_offset, shard_size in shard_offsets:
499 + # Special case for Quantized Weights.
500 # If quantized, we need to adjust the offset and size to account
501 # for the packing.
502 if packed_dim == output_dim:
503 shard_size = shard_size // param.pack_factor
504 shard_offset = shard_offset // param.pack_factor
505
482 - # If marlin, we need to adjust the offset and size to
483 - # account for the tiling.
506 + # Special case for Marlin.
507 shard_size, shard_offset = adjust_marlin_shard(
508 param, shard_size, shard_offset)
509
@@ -502,6 +525,7 @@ class QKVParallelLinear(ColumnParallelLinear):
525 shard_offset = (self.num_heads +
526 self.num_kv_heads) * self.head_size
527 shard_size = self.num_kv_heads * self.head_size
528 + # Special case for Quantized Weights.
529 # If quantized, we need to adjust the offset and size to account
530 # for the packing.
531 packed_dim = getattr(param, "packed_dim", None)
@@ -509,8 +533,7 @@ class QKVParallelLinear(ColumnParallelLinear):
533 shard_size = shard_size // param.pack_factor
534 shard_offset = shard_offset // param.pack_factor
535
512 - # If marlin, we need to adjust the offset and size to
513 - # account for the tiling.
536 + # Special case for Marlin.
537 shard_size, shard_offset = adjust_marlin_shard(
538 param, shard_size, shard_offset)
539
@@ -523,12 +546,17 @@ class QKVParallelLinear(ColumnParallelLinear):
546 start_idx = shard_id * shard_size
547 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
548 shard_size)
549 + # Special case for for AQLM codebooks.
550 elif is_metadata:
551 # metadata indicates fixed size concatenated along dim 0
552 shard_size = loaded_weight.shape[0]
553 shard_index = ["q", "k", "v"].index(loaded_shard_id)
554 param_data = param_data.narrow(0, shard_index * shard_size,
555 shard_size)
556 + # Special case for Fp8 scales.
557 + elif fp8_scales_shard_indexer is not None:
558 + param_data, loaded_weight = fp8_scales_shard_indexer(
559 + param_data, loaded_weight, loaded_shard_id)
560 else:
561 ignore_warning = getattr(param, "ignore_warning", False)
562 if not ignore_warning:
@@ -611,6 +639,10 @@ class RowParallelLinear(LinearBase):
639 self.register_parameter("bias", None)
640
641 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
642 + # Special case for Fp8 scales.
643 + fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
644 + None)
645 +
646 tp_rank = get_tensor_model_parallel_rank()
647 input_dim = getattr(param, "input_dim", None)
648 param_data = param.data
@@ -619,6 +651,12 @@ class RowParallelLinear(LinearBase):
651 start_idx = tp_rank * shard_size
652 loaded_weight = loaded_weight.narrow(input_dim, start_idx,
653 shard_size)
654 + # Special case for Fp8 scales.
655 + elif fp8_scales_shard_indexer is not None:
656 + param_data, loaded_weight = fp8_scales_shard_indexer(param_data,
657 + loaded_weight,
658 + shard_id=0)
659 +
660 assert param_data.shape == loaded_weight.shape
661 param_data.copy_(loaded_weight)
662
vllm/model_executor/layers/quantization/fp8.py
+169 -30
@@ -1,23 +1,36 @@
1 -from typing import Any, Dict, List, Optional
1 +from typing import Any, Dict, List, Optional, Tuple, Union
2
3 import torch
4 from torch.nn import Module
5 from torch.nn.parameter import Parameter
6
7 from vllm import _custom_ops as ops
8 +from vllm.logger import init_logger
9 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10 from vllm.model_executor.layers.quantization.base_config import (
10 - QuantizationConfig, QuantizeMethodBase)
11 + QuantizationConfig)
12 from vllm.model_executor.utils import set_weight_attrs
13
14 +ACTIVATION_SCHEMES = ["static", "dynamic"]
15 +
16 +logger = init_logger(__name__)
17 +
18
19 class Fp8Config(QuantizationConfig):
20 """Config class for FP8."""
21
22 def __init__(
23 self,
24 + is_checkpoint_fp8_serialized: bool = False,
25 activation_scheme: str = "dynamic",
26 ) -> None:
27 + self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized
28 + if is_checkpoint_fp8_serialized:
29 + logger.warning("Detected fp8 checkpoint. Please note that the "
30 + "format is experimental and subject to change.")
31 + if activation_scheme not in ACTIVATION_SCHEMES:
32 + raise ValueError(
33 + f"Unsupported activation scheme {activation_scheme}")
34 self.activation_scheme = activation_scheme
35
36 @classmethod
@@ -30,10 +43,7 @@ class Fp8Config(QuantizationConfig):
43
44 @classmethod
45 def get_min_capability(cls) -> int:
33 - # TODO: PyTorch 2.3.0+ is required to run FP8 on
34 - # SM 89 (e.g. Ada) GPUs. Specifically, this PR has to
35 - # be included: https://github.com/pytorch/pytorch/pull/118881
36 - return 90
46 + return 89
47
48 @classmethod
49 def get_config_filenames(cls) -> List[str]:
@@ -41,11 +51,14 @@ class Fp8Config(QuantizationConfig):
51
52 @classmethod
53 def from_config(cls, config: Dict[str, Any]) -> "Fp8Config":
54 + quant_method = cls.get_from_keys(config, ["quant_method"])
55 + is_checkpoint_fp8_serialized = ("fp8" in quant_method)
56 activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
45 - return cls(activation_scheme)
57 + return cls(is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
58 + activation_scheme=activation_scheme)
59
60 def get_quant_method(
48 - self, layer: torch.nn.Module) -> Optional["QuantizeMethodBase"]:
61 + self, layer: torch.nn.Module) -> Optional["Fp8LinearMethod"]:
62 if isinstance(layer, LinearBase):
63 return Fp8LinearMethod(self)
64 return None
@@ -56,8 +69,12 @@ class Fp8Config(QuantizationConfig):
69
70 class Fp8LinearMethod(LinearMethodBase):
71 """Linear method for FP8.
59 - We now support common FP16/BF16 model checkpoints ONLY. The weight
60 - scaling factor will be initialized after the model weights are loaded.
72 + Supports loading FP8 checkpoints with static weight scale and
73 + dynamic/static activation scale.
74 +
75 + Also supports loading quantized FP16/BF16 model checkpoints with dynamic
76 + activation scaling. The weight scaling factor will be initialized after
77 + the model weights are loaded.
78
79 Limitations:
80 1. Only support per-tensor quantization due to torch._scaled_mm support.
@@ -71,6 +88,24 @@ class Fp8LinearMethod(LinearMethodBase):
88 def __init__(self, quant_config: Fp8Config):
89 self.quant_config = quant_config
90
91 + def _create_scale_param(
92 + self,
93 + scale_name: str,
94 + layer: torch.nn.Module,
95 + output_partition_sizes: List[int],
96 + **extra_weight_attrs,
97 + ) -> None:
98 + scale = Parameter(torch.empty(len(output_partition_sizes),
99 + dtype=torch.float32),
100 + requires_grad=False)
101 + layer.register_parameter(scale_name, scale)
102 + set_weight_attrs(
103 + scale, {
104 + **extra_weight_attrs,
105 + "fp8_scales_shard_indexer":
106 + self.scales_shard_indexer,
107 + })
108 +
109 def create_weights(
110 self,
111 layer: torch.nn.Module,
@@ -81,46 +116,150 @@ class Fp8LinearMethod(LinearMethodBase):
116 params_dtype: torch.dtype,
117 **extra_weight_attrs,
118 ):
119 + del input_size, output_size
120 output_size_per_partition = sum(output_partition_sizes)
121 +
122 + layer.process_after_load = True
123 + layer.logical_widths = output_partition_sizes
124 +
125 + # WEIGHT
126 + weight_dtype = (torch.float8_e4m3fn
127 + if self.quant_config.is_checkpoint_fp8_serialized else
128 + params_dtype)
129 weight = Parameter(torch.empty(output_size_per_partition,
130 input_size_per_partition,
87 - dtype=params_dtype),
131 + dtype=weight_dtype),
132 requires_grad=False)
133 layer.register_parameter("weight", weight)
90 - set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
91 - set_weight_attrs(weight, extra_weight_attrs)
134 + set_weight_attrs(weight, {
135 + **extra_weight_attrs,
136 + "input_dim": 1,
137 + "output_dim": 0,
138 + })
139
93 - w_scale = Parameter(
94 - torch.empty(1, dtype=torch.float32),
95 - requires_grad=False,
96 - )
97 - layer.register_parameter("weight_scaling_factor", w_scale)
140 + # If checkpoint is serialized fp8, load them.
141 + # Otherwise, wait until process_weights_after_loading.
142 + if self.quant_config.is_checkpoint_fp8_serialized:
143 + # WEIGHT SCALE
144 + self._create_scale_param(
145 + scale_name="weight_scale",
146 + layer=layer,
147 + output_partition_sizes=output_partition_sizes,
148 + **extra_weight_attrs)
149 +
150 + # ACTIVATION SCALE
151 + if self.quant_config.activation_scheme == "static":
152 + self._create_scale_param(
153 + scale_name="act_scale",
154 + layer=layer,
155 + output_partition_sizes=output_partition_sizes,
156 + **extra_weight_attrs)
157 +
158 + def scales_shard_indexer(
159 + self, param: torch.Tensor, loaded_weight: torch.Tensor,
160 + shard_id: Union[str, int]) -> Tuple[torch.Tensor, torch.Tensor]:
161 + qkv_idxs = {"q": 0, "k": 1, "v": 2}
162 +
163 + if isinstance(shard_id, int):
164 + pass
165 + elif isinstance(shard_id, str):
166 + if shard_id not in qkv_idxs:
167 + raise ValueError(f"Unknown shard_id: {shard_id}")
168 + shard_id = qkv_idxs[shard_id]
169 + else:
170 + ValueError(f"Shard id must be int or str but got {type(shard_id)}")
171 +
172 + return param[shard_id], loaded_weight
173
174 def process_weights_after_loading(self, layer: Module) -> None:
100 - # Although the quant_method is propagated to all layers,
101 - # only linear layers invoke "create_weights". So we check
102 - # whether "weight_scaling_facor" is registered to determine
103 - # whether the layer is a linear layer that requires quantization.
104 - if not hasattr(layer, "weight_scaling_factor"):
175 + if (not hasattr(layer, "process_after_load")
176 + or not layer.process_after_load):
177 + return
178 +
179 + # If checkpoint is fp/bf16 (not serialized fp8), quantize the weights.
180 + if not self.quant_config.is_checkpoint_fp8_serialized:
181 + qweight, weight_scale = ops.scaled_fp8_quant(layer.weight,
182 + scale=None)
183 + layer.weight = Parameter(qweight.t(), requires_grad=False)
184 + layer.weight_scale = Parameter(weight_scale, requires_grad=False)
185 + layer.logical_widths = None
186 + layer.act_scale = None
187 return
188
107 - qweight, weight_scale = ops.scaled_fp8_quant(layer.weight)
108 - # torch._scaled_mm requires column-major in the second
109 - # input (weight), so we transpose the quantized weight.
110 - layer.weight = Parameter(qweight.t(), requires_grad=False)
111 - layer.weight_scaling_factor.data.copy_(weight_scale)
189 + # If checkpoint is fp8, requantize the separately quantized logical
190 + # weights into a single fp8 weight with a single weight scale.
191 + else:
192 + # WEIGHT_SCALE / WEIGHT
193 + # Loop over logical weights, requantizing with single scale.
194 + max_w_scale = layer.weight_scale.max()
195 + start = 0
196 + for idx, logical_width in enumerate(layer.logical_widths):
197 + end = start + logical_width
198 + weight_dq = per_tensor_dequantize(layer.weight[start:end, :],
199 + layer.weight_scale[idx])
200 +
201 + layer.weight[start:end, :] = per_tensor_quantize(
202 + weight_dq, layer.weight_scale.max())
203 + start = end
204 + layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
205 +
206 + # WEIGHT
207 + # Transpose weight for passing to torch._scaled_mm
208 + weight = layer.weight
209 + layer.weight = Parameter(weight.t(), requires_grad=False)
210 +
211 + # ACT_SCALE
212 + # Dynamic: set to None (required input to ops.scaled_fp8_quant).
213 + # Static: set to max of the act_scales (since they are equal).
214 + if self.quant_config.activation_scheme == "dynamic":
215 + layer.act_scale = None
216 + elif self.quant_config.activation_scheme == "static":
217 + if not all_close_1d(layer.act_scale):
218 + raise ValueError(
219 + "All the act_scales for the logical weights of a layer "
220 + f"must be equal. But got {layer.act_scale}")
221 + layer.act_scale = Parameter(layer.act_scale.max(),
222 + requires_grad=False)
223 + else:
224 + raise ValueError(
225 + f"Unknown scheme {self.quant_config.activation_scheme}")
226
227 def apply(self,
228 layer: torch.nn.Module,
229 x: torch.Tensor,
230 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
117 - qinput, x_scale = ops.scaled_fp8_quant(x)
231 + # ops.scaled_fp8_quant supports both dynamic and static quant.
232 + # If dynamic, layer.act_scale is None and x_scale computed from x.
233 + # If static, layer.act_scale is scalar and x_scale set to act_scale.
234 + qinput, x_scale = ops.scaled_fp8_quant(x, layer.act_scale)
235 +
236 + # Fused GEMM_DQ
237 output, _ = torch._scaled_mm(
238 qinput,
239 layer.weight,
240 out_dtype=x.dtype,
241 scale_a=x_scale,
123 - scale_b=layer.weight_scaling_factor,
242 + scale_b=layer.weight_scale,
243 bias=bias,
244 )
245 +
246 return output
247 +
248 +
249 +def all_close_1d(x: torch.Tensor) -> bool:
250 + assert len(x.shape) == 1
251 + return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
252 +
253 +
254 +def per_tensor_quantize(tensor: torch.Tensor,
255 + inv_scale: float) -> torch.Tensor:
256 + finfo = torch.finfo(torch.float8_e4m3fn)
257 + qweight = (tensor / inv_scale).clamp(min=finfo.min, max=finfo.max)
258 + return qweight.to(torch.float8_e4m3fn)
259 +
260 +
261 +def per_tensor_dequantize(tensor: torch.Tensor,
262 + inv_scale: float) -> torch.Tensor:
263 + fake_qweight = tensor.to(torch.float16)
264 + dq_weight = fake_qweight * inv_scale
265 + return dq_weight