A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Bugfix] Fix quark fp8 format loading on AMD GPUs (#12612)
Signed-off-by: Felix Marty <felmarty@amd.com> Signed-off-by: kewang2 <kewang2@amd.com> Co-authored-by: kewang2 <kewang2@amd.com>
fxmarty-amd committed
May 8, 2025 at 11:53 UTC
bb239a730f2a5534e1d68caf093358b5506f5555
2 files changed
+38
-9
tests/quantization/test_quark.py
+26
@@ -5,6 +5,7 @@ Run `pytest tests/quantization/test_quark.py`.
5
"""
6
7
import pytest
8
+import torch
9
10
from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
11
QuarkLinearMethod, QuarkW8A8Fp8, QuarkW8A8Int8)
@@ -63,3 +64,28 @@ def test_quark_int8_w_per_tensor_a_per_tensor(vllm_runner, tp):
64
65
output = llm.generate_greedy("Hello my name is", max_tokens=20)
66
assert output
67
+
68
+
69
+def test_quark_fp8_parity(vllm_runner):
70
+ quark_model_id = "amd-quark/llama-tiny-fp8-quark-quant-method"
71
+ fp8_model_id = "amd-quark/llama-tiny-fp8-quant-method"
72
+
73
+ llm_kwargs = {
74
+ "tensor_parallel_size": 1,
75
+ "enforce_eager": True,
76
+ "gpu_memory_utilization": 0.1
77
+ }
78
+ with (vllm_runner(quark_model_id, **llm_kwargs) as
79
+ quark_handle, vllm_runner(fp8_model_id, **llm_kwargs) as fp8_handle):
80
+ quark_model = (quark_handle.model.llm_engine.model_executor.
81
+ driver_worker.model_runner.model)
82
+ quark_state_dict = quark_model.state_dict()
83
+
84
+ fp8_model = (fp8_handle.model.llm_engine.model_executor.driver_worker.
85
+ model_runner.model)
86
+ fp8_state_dict = fp8_model.state_dict()
87
+
88
+ assert fp8_state_dict.keys() == quark_state_dict.keys()
89
+
90
+ for key in fp8_state_dict:
91
+ assert torch.equal(fp8_state_dict[key], quark_state_dict[key])
vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py
+12
-9
@@ -34,21 +34,24 @@ class QuarkW8A8Fp8(QuarkScheme):
34
# tensor scales (thus N scales being passed to the kernel),
35
# requantize so we can always run per tensor
36
if self.qscheme == "per_tensor":
37
- max_w_scale, weight = requantize_with_max_scale(
38
- weight=layer.weight,
39
- weight_scale=layer.weight_scale,
40
- logical_widths=layer.logical_widths,
41
- )
42
-
43
- if current_platform.is_fp8_fnuz():
37
+ if current_platform.is_rocm():
38
input_scale = getattr(layer, 'input_scale', None)
39
weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
46
- weight=weight,
47
- weight_scale=max_w_scale,
40
+ weight=layer.weight,
41
+ weight_scale=layer.weight_scale,
42
input_scale=input_scale)
43
if input_scale is not None:
44
layer.input_scale = Parameter(input_scale,
45
requires_grad=False)
46
+ else:
47
+ max_w_scale = layer.weight_scale
48
+ weight = layer.weight
49
+
50
+ max_w_scale, weight = requantize_with_max_scale(
51
+ weight=weight,
52
+ weight_scale=max_w_scale,
53
+ logical_widths=layer.logical_widths,
54
+ )
55
56
layer.weight = Parameter(weight.t(), requires_grad=False)
57
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)