A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Model] [Quantization] Support deepseek_v3 w8a8 fp8 block-wise quantization (#11523)
Signed-off-by: mgoin <michael@neuralmagic.com> Signed-off-by: simon-mo <simon.mo@hey.com> Signed-off-by: simon-mo <xmo@berkeley.edu> Co-authored-by: simon-mo <simon.mo@hey.com> Co-authored-by: simon-mo <xmo@berkeley.edu> Co-authored-by: HandH1998 <1335248067@qq.com>
Michael Goin committed
Dec 26, 2024 at 18:33 UTC
2072924d1480460d3b3578a4548c2bffe33fe1c3
8 files changed
+931
-70
tests/kernels/test_block_fp8.py
new
+265
@@ -0,0 +1,265 @@
1
+# Adapted from https://github.com/sgl-project/sglang/pull/2575
2
+import itertools
3
+
4
+import pytest
5
+import torch
6
+
7
+from vllm.model_executor.layers.activation import SiluAndMul
8
+from vllm.model_executor.layers.fused_moe import fused_moe
9
+from vllm.model_executor.layers.quantization.utils.fp8_utils import (
10
+ per_token_group_quant_fp8, w8a8_block_fp8_matmul)
11
+from vllm.platforms import current_platform
12
+
13
+if current_platform.get_device_capability() < (9, 0):
14
+ pytest.skip("FP8 Triton requires CUDA 9.0 or higher",
15
+ allow_module_level=True)
16
+
17
+# Test configurations
18
+DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
19
+NUM_TOKENS = [7, 83, 2048]
20
+D = [512, 4096, 5120, 13824]
21
+GROUP_SIZE = [64, 128, 256, 512]
22
+M = [1, 7, 83, 512, 2048]
23
+N = [128, 512, 1024, 4096, 7748, 13824]
24
+K = [256, 4096, 5120, 3884, 13824]
25
+# Deepseek-V3's intermediate size 18432, so N is 18432*2/8=4608 at TP8
26
+# and its hidden size is 7168.
27
+M_moe = [1, 7, 83, 512, 2048]
28
+N_moe = [4608] # [128, 4608, 13824]
29
+K_moe = [7168] # [256, 7168, 13824]
30
+BLOCK_SIZE = [[128, 128]]
31
+E = [256] # [8, 24, 128, 256]
32
+TOP_KS = [1] # [1, 2, 6]
33
+OUT_DTYPES = [torch.bfloat16] # [torch.float32, torch.half, torch.bfloat16]
34
+SEEDS = [0]
35
+
36
+
37
+def native_per_token_group_quant_fp8(x,
38
+ group_size,
39
+ eps=1e-10,
40
+ dtype=torch.float8_e4m3fn):
41
+ """Function to perform per-token-group quantization on an input tensor
42
+ `x` using native torch."""
43
+ assert x.shape[-1] % group_size == 0, ("the last dimension of `x` cannot "
44
+ "be divisible by `group_size`")
45
+ assert x.is_contiguous(), "`x` is not contiguous"
46
+
47
+ finfo = torch.finfo(dtype)
48
+ fp8_min = finfo.min
49
+ fp8_max = finfo.max
50
+
51
+ x_ = x.reshape(x.numel() // group_size, group_size)
52
+ amax = x_.abs().max(dim=-1,
53
+ keepdim=True)[0].clamp(min=eps).to(torch.float32)
54
+ x_s = amax / fp8_max
55
+ x_q = (x_ / x_s).clamp(min=fp8_min, max=fp8_max).to(dtype)
56
+ x_q = x_q.reshape(x.shape)
57
+ x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size, ))
58
+
59
+ return x_q, x_s
60
+
61
+
62
+def native_w8a8_block_fp8_matmul(A,
63
+ B,
64
+ As,
65
+ Bs,
66
+ block_size,
67
+ output_dtype=torch.float16):
68
+ """Matrix multiplication with block-wise quantization using native torch."""
69
+ A = A.to(torch.float32)
70
+ B = B.to(torch.float32)
71
+ assert A.shape[-1] == B.shape[-1]
72
+ assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2
73
+ assert len(block_size) == 2
74
+ block_n, block_k = block_size[0], block_size[1]
75
+ assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1]
76
+ assert A.shape[:-1] == As.shape[:-1]
77
+
78
+ M = A.numel() // A.shape[-1]
79
+ N, K = B.shape
80
+ origin_C_shape = A.shape[:-1] + (N, )
81
+ A = A.reshape(M, A.shape[-1])
82
+ As = As.reshape(M, As.shape[-1])
83
+ n_tiles = (N + block_n - 1) // block_n
84
+ k_tiles = (K + block_k - 1) // block_k
85
+ assert n_tiles == Bs.shape[0]
86
+ assert k_tiles == Bs.shape[1]
87
+
88
+ C_shape = (M, N)
89
+ C = torch.zeros(C_shape, dtype=torch.float32, device=A.device)
90
+
91
+ A_tiles = [
92
+ A[:, i * block_k:min((i + 1) * block_k, K)] for i in range(k_tiles)
93
+ ]
94
+ B_tiles = [[
95
+ B[j * block_n:min((j + 1) * block_n, N),
96
+ i * block_k:min((i + 1) * block_k, K), ] for i in range(k_tiles)
97
+ ] for j in range(n_tiles)]
98
+ C_tiles = [
99
+ C[:, j * block_n:min((j + 1) * block_n, N)] for j in range(n_tiles)
100
+ ]
101
+ As_tiles = [As[:, i:i + 1] for i in range(k_tiles)]
102
+
103
+ for i in range(k_tiles):
104
+ for j in range(n_tiles):
105
+ a = A_tiles[i]
106
+ b = B_tiles[j][i]
107
+ c = C_tiles[j]
108
+ s = As_tiles[i] * Bs[j][i]
109
+ c[:, :] += torch.matmul(a, b.t()) * s
110
+
111
+ C = C.reshape(origin_C_shape).to(output_dtype)
112
+ return C
113
+
114
+
115
+def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
116
+ """Fused moe with block-wise quantization using native torch."""
117
+ B, D = a.shape
118
+ a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
119
+ out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
120
+ score = torch.softmax(score, dim=-1, dtype=torch.float32)
121
+ topk_weight, topk_ids = torch.topk(score, topk)
122
+ topk_weight = topk_weight.view(-1)
123
+ topk_ids = topk_ids.view(-1)
124
+
125
+ _, block_k = block_shape[0], block_shape[1]
126
+ a_q, a_s = native_per_token_group_quant_fp8(a, block_k)
127
+ a_q = a_q.to(torch.float32)
128
+ for i in range(w1.shape[0]):
129
+ mask = topk_ids == i
130
+ if mask.sum():
131
+ inter_out = native_w8a8_block_fp8_matmul(a_q[mask],
132
+ w1[i],
133
+ a_s[mask],
134
+ w1_s[i],
135
+ block_shape,
136
+ output_dtype=a.dtype)
137
+ act_out = SiluAndMul().forward_native(inter_out)
138
+ act_out_q, act_out_s = native_per_token_group_quant_fp8(
139
+ act_out, block_k)
140
+ act_out = act_out.to(torch.float32)
141
+ out[mask] = native_w8a8_block_fp8_matmul(act_out_q,
142
+ w2[i],
143
+ act_out_s,
144
+ w2_s[i],
145
+ block_shape,
146
+ output_dtype=a.dtype)
147
+ return (out.view(B, -1, w2.shape[1]) *
148
+ topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
149
+
150
+
151
+# Skip all tests if CUDA is not available
152
+pytest.importorskip("torch.cuda")
153
+
154
+
155
+@pytest.fixture(autouse=True)
156
+def setup_cuda():
157
+ torch.set_default_device("cuda")
158
+
159
+
160
+@pytest.mark.parametrize("num_tokens,d,dtype,group_size,seed",
161
+ itertools.product(NUM_TOKENS, D, DTYPES, GROUP_SIZE,
162
+ SEEDS))
163
+@torch.inference_mode()
164
+def test_per_token_group_quant_fp8(num_tokens, d, dtype, group_size, seed):
165
+ torch.manual_seed(seed)
166
+ x = torch.rand(num_tokens, d, dtype=dtype)
167
+
168
+ ref_out, ref_scale = native_per_token_group_quant_fp8(x, group_size)
169
+ out, scale = per_token_group_quant_fp8(x, group_size)
170
+
171
+ assert torch.allclose(out.to(torch.float32),
172
+ ref_out.to(torch.float32),
173
+ rtol=0.15)
174
+ assert torch.allclose(scale, ref_scale)
175
+
176
+
177
+@pytest.mark.parametrize("M,N,K,block_size,out_dtype,seed",
178
+ itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES,
179
+ SEEDS))
180
+@torch.inference_mode()
181
+def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed):
182
+ torch.manual_seed(seed)
183
+ factor_for_scale = 1e-2
184
+ fp8_info = torch.finfo(torch.float8_e4m3fn)
185
+ fp8_max, fp8_min = fp8_info.max, fp8_info.min
186
+
187
+ A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
188
+ A_fp8 = A_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
189
+
190
+ B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
191
+ B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
192
+
193
+ block_n, block_k = block_size[0], block_size[1]
194
+ n_tiles = (N + block_n - 1) // block_n
195
+ k_tiles = (K + block_k - 1) // block_k
196
+
197
+ As = torch.rand(M, k_tiles, dtype=torch.float32) * factor_for_scale
198
+ Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
199
+
200
+ ref_out = native_w8a8_block_fp8_matmul(A_fp8, B_fp8, As, Bs, block_size,
201
+ out_dtype)
202
+ out = w8a8_block_fp8_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
203
+
204
+ rel_diff = (torch.mean(
205
+ torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) /
206
+ torch.mean(torch.abs(ref_out.to(torch.float32))))
207
+ assert rel_diff < 0.001
208
+
209
+
210
+@pytest.mark.parametrize("M,N,K,E,topk,block_size,dtype,seed",
211
+ itertools.product(M_moe, N_moe, K_moe, E, TOP_KS,
212
+ BLOCK_SIZE, DTYPES, SEEDS))
213
+@torch.inference_mode()
214
+def test_w8a8_block_fp8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
215
+ torch.manual_seed(seed)
216
+ factor_for_scale = 1e-2
217
+ fp8_info = torch.finfo(torch.float8_e4m3fn)
218
+ fp8_max, fp8_min = fp8_info.max, fp8_info.min
219
+
220
+ a = torch.randn((M, K), dtype=dtype) / 10
221
+
222
+ w1_bf16 = (torch.rand(
223
+ (E, 2 * N, K), dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
224
+ w1 = w1_bf16.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
225
+ del w1_bf16
226
+
227
+ w2_bf16 = (torch.rand((E, K, N), dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
228
+ w2 = w2_bf16.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
229
+ del w2_bf16
230
+
231
+ block_n, block_k = block_size[0], block_size[1]
232
+ n_tiles_w1 = (2 * N + block_n - 1) // block_n
233
+ n_tiles_w2 = (K + block_n - 1) // block_n
234
+ k_tiles_w1 = (K + block_k - 1) // block_k
235
+ k_tiles_w2 = (N + block_k - 1) // block_k
236
+
237
+ w1_s = torch.rand(
238
+ (E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) * factor_for_scale
239
+ w2_s = torch.rand(
240
+ (E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) * factor_for_scale
241
+
242
+ score = torch.randn((M, E), dtype=dtype)
243
+
244
+ out = fused_moe(
245
+ a,
246
+ w1,
247
+ w2,
248
+ score,
249
+ topk,
250
+ renormalize=False,
251
+ use_fp8_w8a8=True,
252
+ w1_scale=w1_s,
253
+ w2_scale=w2_s,
254
+ block_shape=block_size,
255
+ )
256
+ ref_out = torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, score, topk,
257
+ block_size)
258
+
259
+ print(f"{out.sum()=}")
260
+ print(f"{ref_out.sum()=}")
261
+
262
+ rel_diff = (torch.mean(
263
+ torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) /
264
+ torch.mean(torch.abs(ref_out.to(torch.float32))))
265
+ assert rel_diff < 0.03
vllm/config.py
+7
-7
@@ -161,7 +161,7 @@ class ModelConfig:
161
override default pooling config for the pooling model.
162
logits_processor_pattern: Optional regex pattern specifying valid
163
logits processor qualified names that can be passed with the
164
- `logits_processors` extra completion argument. Defaults to None,
164
+ `logits_processors` extra completion argument. Defaults to None,
165
which allows no processors.
166
generation_config: Configuration parameter file for generation.
167
"""
@@ -364,7 +364,7 @@ class ModelConfig:
364
def maybe_pull_model_tokenizer_for_s3(self, model: str,
365
tokenizer: str) -> None:
366
"""
367
- Pull the model config or tokenizer to a temporary
367
+ Pull the model config or tokenizer to a temporary
368
directory in case of S3.
369
370
Args:
@@ -866,14 +866,14 @@ class ModelConfig:
866
867
def get_diff_sampling_param(self) -> Dict[str, Any]:
868
"""
869
- This method returns a dictionary containing the parameters
870
- that differ from the default sampling parameters, but only
871
- if `generation_config` is set. If `generation_config` is not
869
+ This method returns a dictionary containing the parameters
870
+ that differ from the default sampling parameters, but only
871
+ if `generation_config` is set. If `generation_config` is not
872
set, an empty dictionary is returned.
873
874
Returns:
875
- Dict[str, Any]: A dictionary with the differing sampling
876
- parameters if `generation_config` is set, otherwise an
875
+ Dict[str, Any]: A dictionary with the differing sampling
876
+ parameters if `generation_config` is set, otherwise an
877
empty dictionary.
878
"""
879
if self.generation_config is None:
vllm/model_executor/layers/fused_moe/fused_moe.py
+103
-28
@@ -2,7 +2,7 @@
2
import functools
3
import json
4
import os
5
-from typing import Any, Callable, Dict, Optional, Tuple
5
+from typing import Any, Callable, Dict, List, Optional, Tuple
6
7
import torch
8
import triton
@@ -11,6 +11,8 @@ import triton.language as tl
11
import vllm.envs as envs
12
from vllm import _custom_ops as ops
13
from vllm.logger import init_logger
14
+from vllm.model_executor.layers.quantization.utils.fp8_utils import (
15
+ per_token_group_quant_fp8)
16
from vllm.platforms import current_platform
17
from vllm.utils import direct_register_custom_op
18
@@ -45,8 +47,14 @@ def fused_moe_kernel(
47
stride_bn,
48
stride_cm,
49
stride_cn,
50
+ stride_asm,
51
+ stride_ask,
52
stride_bse,
53
+ stride_bsk,
54
stride_bsn,
55
+ # Block size for block-wise quantization
56
+ group_n: tl.constexpr,
57
+ group_k: tl.constexpr,
58
# Meta-parameters
59
BLOCK_SIZE_M: tl.constexpr,
60
BLOCK_SIZE_N: tl.constexpr,
@@ -125,8 +133,14 @@ def fused_moe_kernel(
133
b_scale = tl.load(b_scale_ptrs)
134
135
if use_fp8_w8a8:
128
- a_scale = tl.load(a_scale_ptr)
129
- b_scale = tl.load(b_scale_ptr + off_experts)
136
+ if group_k > 0 and group_n > 0:
137
+ a_scale_ptrs = a_scale_ptr + (offs_token // top_k) * stride_asm
138
+ offs_bsn = offs_bn // group_n
139
+ b_scale_ptrs = (b_scale_ptr + off_experts * stride_bse +
140
+ offs_bsn * stride_bsn)
141
+ else:
142
+ a_scale = tl.load(a_scale_ptr)
143
+ b_scale = tl.load(b_scale_ptr + off_experts)
144
145
# -----------------------------------------------------------
146
# Iterate to compute a block of the C matrix.
@@ -149,7 +163,18 @@ def fused_moe_kernel(
163
if use_int8_w8a16:
164
accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
165
elif use_fp8_w8a8:
152
- accumulator = tl.dot(a, b, acc=accumulator)
166
+ if group_k > 0 and group_n > 0:
167
+ k_start = k * BLOCK_SIZE_K
168
+ offs_ks = k_start // group_k
169
+ a_scale = tl.load(a_scale_ptrs + offs_ks * stride_ask,
170
+ mask=token_mask,
171
+ other=0.0)
172
+ b_scale = tl.load(b_scale_ptrs + offs_ks * stride_bsk)
173
+
174
+ accumulator += tl.dot(a, b) * a_scale[:,
175
+ None] * b_scale[None, :]
176
+ else:
177
+ accumulator = tl.dot(a, b, acc=accumulator)
178
else:
179
accumulator += tl.dot(a, b)
180
# Advance the ptrs to the next K block.
@@ -164,7 +189,10 @@ def fused_moe_kernel(
189
if use_int8_w8a16:
190
accumulator = (accumulator * b_scale).to(compute_type)
191
elif use_fp8_w8a8:
167
- accumulator = (accumulator * a_scale * b_scale).to(compute_type)
192
+ if group_k > 0 and group_n > 0:
193
+ accumulator = accumulator.to(compute_type)
194
+ else:
195
+ accumulator = (accumulator * a_scale * b_scale).to(compute_type)
196
else:
197
accumulator = accumulator.to(compute_type)
198
# -----------------------------------------------------------
@@ -233,22 +261,37 @@ def moe_align_block_size(
261
return sorted_ids, expert_ids, num_tokens_post_pad
262
263
236
-def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
264
+def invoke_fused_moe_kernel(A: torch.Tensor,
265
+ B: torch.Tensor,
266
+ C: torch.Tensor,
267
A_scale: Optional[torch.Tensor],
268
B_scale: Optional[torch.Tensor],
239
- topk_weights: torch.Tensor, topk_ids: torch.Tensor,
269
+ topk_weights: torch.Tensor,
270
+ topk_ids: torch.Tensor,
271
sorted_token_ids: torch.Tensor,
272
expert_ids: torch.Tensor,
273
num_tokens_post_padded: torch.Tensor,
243
- mul_routed_weight: bool, top_k: int,
244
- config: Dict[str, Any], compute_type: tl.dtype,
245
- use_fp8_w8a8: bool, use_int8_w8a16: bool) -> None:
274
+ mul_routed_weight: bool,
275
+ top_k: int,
276
+ config: Dict[str, Any],
277
+ compute_type: tl.dtype,
278
+ use_fp8_w8a8: bool,
279
+ use_int8_w8a16: bool,
280
+ block_shape: Optional[List[int]] = None) -> None:
281
assert topk_weights.stride(1) == 1
282
assert sorted_token_ids.stride(0) == 1
283
284
if use_fp8_w8a8:
250
- A, A_scale = ops.scaled_fp8_quant(A, A_scale)
285
assert B_scale is not None
286
+ if block_shape is None:
287
+ A, A_scale = ops.scaled_fp8_quant(A, A_scale)
288
+ else:
289
+ assert len(block_shape) == 2
290
+ block_n, block_k = block_shape[0], block_shape[1]
291
+ A, A_scale = per_token_group_quant_fp8(A, block_k)
292
+ assert triton.cdiv(A.shape[-1], block_k) == A_scale.shape[-1]
293
+ assert triton.cdiv(B.shape[-2], block_n) == B_scale.shape[-2]
294
+ assert triton.cdiv(B.shape[-1], block_k) == B_scale.shape[-1]
295
elif use_int8_w8a16:
296
assert B_scale is not None
297
else:
@@ -279,8 +322,13 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
322
B.stride(1),
323
C.stride(1),
324
C.stride(2),
282
- B_scale.stride(0) if B_scale is not None and use_int8_w8a16 else 0,
283
- B_scale.stride(1) if B_scale is not None and use_int8_w8a16 else 0,
325
+ A_scale.stride(0) if A_scale is not None and A_scale.ndim == 2 else 0,
326
+ A_scale.stride(1) if A_scale is not None and A_scale.ndim == 2 else 0,
327
+ B_scale.stride(0) if B_scale is not None and B_scale.ndim >= 2 else 0,
328
+ B_scale.stride(2) if B_scale is not None and B_scale.ndim == 3 else 0,
329
+ B_scale.stride(1) if B_scale is not None and B_scale.ndim >= 2 else 0,
330
+ 0 if block_shape is None else block_shape[0],
331
+ 0 if block_shape is None else block_shape[1],
332
MUL_ROUTED_WEIGHT=mul_routed_weight,
333
top_k=top_k,
334
compute_type=compute_type,
@@ -362,6 +410,7 @@ def try_get_optimal_moe_config(
410
dtype: Optional[str],
411
M: int,
412
is_marlin: bool = False,
413
+ block_shape: Optional[List[int]] = None,
414
):
415
from vllm.model_executor.layers.fused_moe import get_config
416
override_config = get_config()
@@ -380,6 +429,12 @@ def try_get_optimal_moe_config(
429
# Else use the default config
430
config = get_default_config(M, E, N, w1_shape[2], top_k, dtype,
431
is_marlin)
432
+ # NOTE: For block-wise quant,
433
+ # BLOCK_K must be divisible by block_shape[1]
434
+ # BLOCK_N and BLOCK_M has no requirements
435
+ if block_shape is not None:
436
+ config["BLOCK_SIZE_N"] = block_shape[0]
437
+ config["BLOCK_SIZE_K"] = block_shape[1]
438
return config
439
440
@@ -479,10 +534,11 @@ def inplace_fused_experts(hidden_states: torch.Tensor,
534
w1_scale: Optional[torch.Tensor] = None,
535
w2_scale: Optional[torch.Tensor] = None,
536
a1_scale: Optional[torch.Tensor] = None,
482
- a2_scale: Optional[torch.Tensor] = None) -> None:
537
+ a2_scale: Optional[torch.Tensor] = None,
538
+ block_shape: Optional[List[int]] = None) -> None:
539
fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids, True,
540
use_fp8_w8a8, use_int8_w8a16, w1_scale, w2_scale,
485
- a1_scale, a2_scale)
541
+ a1_scale, a2_scale, block_shape)
542
543
544
def inplace_fused_experts_fake(
@@ -496,7 +552,8 @@ def inplace_fused_experts_fake(
552
w1_scale: Optional[torch.Tensor] = None,
553
w2_scale: Optional[torch.Tensor] = None,
554
a1_scale: Optional[torch.Tensor] = None,
499
- a2_scale: Optional[torch.Tensor] = None) -> None:
555
+ a2_scale: Optional[torch.Tensor] = None,
556
+ block_shape: Optional[List[int]] = None) -> None:
557
pass
558
559
@@ -519,10 +576,11 @@ def outplace_fused_experts(
576
w1_scale: Optional[torch.Tensor] = None,
577
w2_scale: Optional[torch.Tensor] = None,
578
a1_scale: Optional[torch.Tensor] = None,
522
- a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
579
+ a2_scale: Optional[torch.Tensor] = None,
580
+ block_shape: Optional[List[int]] = None) -> torch.Tensor:
581
return fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids,
582
False, use_fp8_w8a8, use_int8_w8a16, w1_scale,
525
- w2_scale, a1_scale, a2_scale)
583
+ w2_scale, a1_scale, a2_scale, block_shape)
584
585
586
def outplace_fused_experts_fake(
@@ -536,7 +594,8 @@ def outplace_fused_experts_fake(
594
w1_scale: Optional[torch.Tensor] = None,
595
w2_scale: Optional[torch.Tensor] = None,
596
a1_scale: Optional[torch.Tensor] = None,
539
- a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
597
+ a2_scale: Optional[torch.Tensor] = None,
598
+ block_shape: Optional[List[int]] = None) -> torch.Tensor:
599
return torch.empty_like(hidden_states)
600
601
@@ -559,18 +618,22 @@ def fused_experts(hidden_states: torch.Tensor,
618
w1_scale: Optional[torch.Tensor] = None,
619
w2_scale: Optional[torch.Tensor] = None,
620
a1_scale: Optional[torch.Tensor] = None,
562
- a2_scale: Optional[torch.Tensor] = None):
621
+ a2_scale: Optional[torch.Tensor] = None,
622
+ block_shape: Optional[List[int]] = None):
623
if inplace:
624
torch.ops.vllm.inplace_fused_experts(hidden_states, w1, w2,
625
topk_weights, topk_ids,
626
use_fp8_w8a8, use_int8_w8a16,
627
w1_scale, w2_scale, a1_scale,
568
- a2_scale)
628
+ a2_scale, block_shape)
629
return hidden_states
630
else:
571
- return torch.ops.vllm.outplace_fused_experts(
572
- hidden_states, w1, w2, topk_weights, topk_ids, use_fp8_w8a8,
573
- use_int8_w8a16, w1_scale, w2_scale, a1_scale, a2_scale)
631
+ return torch.ops.vllm.outplace_fused_experts(hidden_states, w1, w2,
632
+ topk_weights, topk_ids,
633
+ use_fp8_w8a8,
634
+ use_int8_w8a16, w1_scale,
635
+ w2_scale, a1_scale,
636
+ a2_scale, block_shape)
637
638
639
def fused_experts_impl(hidden_states: torch.Tensor,
@@ -584,7 +647,8 @@ def fused_experts_impl(hidden_states: torch.Tensor,
647
w1_scale: Optional[torch.Tensor] = None,
648
w2_scale: Optional[torch.Tensor] = None,
649
a1_scale: Optional[torch.Tensor] = None,
587
- a2_scale: Optional[torch.Tensor] = None):
650
+ a2_scale: Optional[torch.Tensor] = None,
651
+ block_shape: Optional[List[int]] = None):
652
# Check constraints.
653
assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
654
assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
@@ -611,6 +675,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
675
w2.shape,
676
topk_ids.shape[1],
677
config_dtype,
678
+ block_shape=block_shape,
679
)
680
681
config = get_config_func(M)
@@ -674,7 +739,8 @@ def fused_experts_impl(hidden_states: torch.Tensor,
739
config,
740
compute_type=compute_type,
741
use_fp8_w8a8=use_fp8_w8a8,
677
- use_int8_w8a16=use_int8_w8a16)
742
+ use_int8_w8a16=use_int8_w8a16,
743
+ block_shape=block_shape)
744
745
ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
746
@@ -693,7 +759,8 @@ def fused_experts_impl(hidden_states: torch.Tensor,
759
config,
760
compute_type=compute_type,
761
use_fp8_w8a8=use_fp8_w8a8,
696
- use_int8_w8a16=use_int8_w8a16)
762
+ use_int8_w8a16=use_int8_w8a16,
763
+ block_shape=block_shape)
764
765
ops.moe_sum(intermediate_cache3.view(*intermediate_cache3.shape),
766
out_hidden_states[begin_chunk_idx:end_chunk_idx])
@@ -718,6 +785,7 @@ def fused_moe(
785
w2_scale: Optional[torch.Tensor] = None,
786
a1_scale: Optional[torch.Tensor] = None,
787
a2_scale: Optional[torch.Tensor] = None,
788
+ block_shape: Optional[List[int]] = None,
789
) -> torch.Tensor:
790
"""
791
This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -745,6 +813,12 @@ def fused_moe(
813
w1.
814
- w2_scale (Optional[torch.Tensor]): Optional scale to be used for
815
w2.
816
+ - a1_scale (Optional[torch.Tensor]): Optional scale to be used for
817
+ a1.
818
+ - a2_scale (Optional[torch.Tensor]): Optional scale to be used for
819
+ a2.
820
+ - block_shape: (Optional[List[int]]): Optional block size for block-wise
821
+ quantization.
822
823
Returns:
824
- torch.Tensor: The output tensor after applying the MoE layer.
@@ -775,4 +849,5 @@ def fused_moe(
849
w1_scale=w1_scale,
850
w2_scale=w2_scale,
851
a1_scale=a1_scale,
778
- a2_scale=a2_scale)
852
+ a2_scale=a2_scale,
853
+ block_shape=block_shape)
vllm/model_executor/layers/fused_moe/layer.py
+6
-1
@@ -29,6 +29,7 @@ class FusedMoeWeightScaleSupported(Enum):
29
TENSOR = "tensor"
30
CHANNEL = "channel"
31
GROUP = "group"
32
+ BLOCK = "block"
33
34
35
class FusedMoEMethodBase(QuantizeMethodBase):
@@ -199,6 +200,7 @@ class FusedMoE(torch.nn.Module):
200
get_tensor_model_parallel_world_size())
201
self.top_k = top_k
202
self.num_experts = num_experts
203
+ assert intermediate_size % self.tp_size == 0
204
self.intermediate_size_per_partition = intermediate_size // self.tp_size
205
self.reduce_results = reduce_results
206
self.renormalize = renormalize
@@ -398,7 +400,10 @@ class FusedMoE(torch.nn.Module):
400
loaded_weight=loaded_weight,
401
expert_data=expert_data,
402
tp_rank=tp_rank)
401
- elif quant_method == FusedMoeWeightScaleSupported.GROUP.value:
403
+ elif quant_method in [
404
+ FusedMoeWeightScaleSupported.GROUP.value,
405
+ FusedMoeWeightScaleSupported.BLOCK.value,
406
+ ]:
407
self._load_model_weight_or_group_weight_scale(
408
shard_id=shard_id,
409
shard_dim=shard_dim,
vllm/model_executor/layers/linear.py
+21
-2
@@ -14,11 +14,14 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
14
from vllm.logger import init_logger
15
from vllm.model_executor.layers.quantization.base_config import (
16
QuantizationConfig, QuantizeMethodBase)
17
+# yapf: disable
18
from vllm.model_executor.parameter import (BasevLLMParameter,
19
+ BlockQuantScaleParameter,
20
PackedColumnParameter,
21
PackedvLLMParameter,
22
PerTensorScaleParameter,
23
RowvLLMParameter)
24
+# yapf: enable
25
from vllm.model_executor.utils import set_weight_attrs
26
27
logger = init_logger(__name__)
@@ -623,8 +626,24 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
626
assert loaded_shard_id < len(self.output_sizes)
627
628
tp_size = get_tensor_model_parallel_world_size()
626
- shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
627
- shard_size = self.output_sizes[loaded_shard_id] // tp_size
629
+
630
+ if isinstance(param, BlockQuantScaleParameter):
631
+ from vllm.model_executor.layers.quantization.fp8 import (
632
+ Fp8LinearMethod, Fp8MoEMethod)
633
+ assert self.quant_method is not None
634
+ assert isinstance(self.quant_method,
635
+ (Fp8LinearMethod, Fp8MoEMethod))
636
+ weight_block_size = self.quant_method.quant_config.weight_block_size
637
+ assert weight_block_size is not None
638
+ block_n, _ = weight_block_size[0], weight_block_size[1]
639
+ shard_offset = (
640
+ (sum(self.output_sizes[:loaded_shard_id]) + block_n - 1) //
641
+ block_n) // tp_size
642
+ shard_size = ((self.output_sizes[loaded_shard_id] + block_n - 1) //
643
+ block_n // tp_size)
644
+ else:
645
+ shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
646
+ shard_size = self.output_sizes[loaded_shard_id] // tp_size
647
648
param.load_merged_column_weight(loaded_weight=loaded_weight,
649
shard_id=loaded_shard_id,
vllm/model_executor/layers/quantization/fp8.py
+167
-32
@@ -6,6 +6,7 @@ from torch.nn.parameter import Parameter
6
7
import vllm.envs as envs
8
from vllm import _custom_ops as ops
9
+from vllm.distributed import get_tensor_model_parallel_world_size
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
12
FusedMoeWeightScaleSupported)
@@ -14,6 +15,8 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
15
from vllm.model_executor.layers.quantization.base_config import (
16
QuantizationConfig, QuantizeMethodBase)
17
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
18
+from vllm.model_executor.layers.quantization.utils.fp8_utils import (
19
+ apply_w8a8_block_fp8_linear)
20
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
21
apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
22
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -22,7 +25,8 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
25
all_close_1d, apply_fp8_linear, convert_to_channelwise,
26
cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
27
requantize_with_max_scale)
25
-from vllm.model_executor.parameter import (ModelWeightParameter,
28
+from vllm.model_executor.parameter import (BlockQuantScaleParameter,
29
+ ModelWeightParameter,
30
PerTensorScaleParameter)
31
from vllm.model_executor.utils import set_weight_attrs
32
from vllm.platforms import current_platform
@@ -41,6 +45,7 @@ class Fp8Config(QuantizationConfig):
45
is_checkpoint_fp8_serialized: bool = False,
46
activation_scheme: str = "dynamic",
47
ignored_layers: Optional[List[str]] = None,
48
+ weight_block_size: Optional[List[int]] = None,
49
) -> None:
50
self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized
51
if is_checkpoint_fp8_serialized:
@@ -51,6 +56,20 @@ class Fp8Config(QuantizationConfig):
56
f"Unsupported activation scheme {activation_scheme}")
57
self.activation_scheme = activation_scheme
58
self.ignored_layers = ignored_layers or []
59
+ if weight_block_size is not None:
60
+ if not is_checkpoint_fp8_serialized:
61
+ raise ValueError(
62
+ "The block-wise quantization only supports fp8-serialized "
63
+ "checkpoint for now.")
64
+ if len(weight_block_size) != 2:
65
+ raise ValueError(
66
+ "The quantization block size of weight must have 2 "
67
+ f"dimensions, but got {len(weight_block_size)} dimensions")
68
+ if activation_scheme != "dynamic":
69
+ raise ValueError("The block-wise quantization only supports "
70
+ "dynamic activation scheme for now, but got "
71
+ f"{activation_scheme} activation scheme.")
72
+ self.weight_block_size = weight_block_size
73
74
@classmethod
75
def get_name(cls) -> str:
@@ -74,9 +93,12 @@ class Fp8Config(QuantizationConfig):
93
is_checkpoint_fp8_serialized = ("fp8" in quant_method)
94
activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
95
ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
96
+ weight_block_size = cls.get_from_keys_or(config, ["weight_block_size"],
97
+ None)
98
return cls(is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
99
activation_scheme=activation_scheme,
79
- ignored_layers=ignored_layers)
100
+ ignored_layers=ignored_layers,
101
+ weight_block_size=weight_block_size)
102
103
def get_quant_method(self, layer: torch.nn.Module,
104
prefix: str) -> Optional["QuantizeMethodBase"]:
@@ -123,6 +145,11 @@ class Fp8LinearMethod(LinearMethodBase):
145
if current_platform.is_rocm():
146
self.use_marlin = False
147
148
+ self.block_quant = self.quant_config.weight_block_size is not None
149
+ if self.block_quant:
150
+ # Marlin doesn't support block-wise fp8
151
+ self.use_marlin = False
152
+
153
def create_weights(
154
self,
155
layer: torch.nn.Module,
@@ -133,10 +160,34 @@ class Fp8LinearMethod(LinearMethodBase):
160
params_dtype: torch.dtype,
161
**extra_weight_attrs,
162
):
136
- del input_size, output_size
163
output_size_per_partition = sum(output_partition_sizes)
164
weight_loader = extra_weight_attrs.get("weight_loader")
165
166
+ if self.block_quant:
167
+ tp_size = get_tensor_model_parallel_world_size()
168
+ assert self.quant_config.weight_block_size is not None
169
+ block_n, block_k = (
170
+ self.quant_config.weight_block_size[0],
171
+ self.quant_config.weight_block_size[1],
172
+ )
173
+ # Required by row parallel
174
+ if (tp_size > 1
175
+ and input_size // input_size_per_partition == tp_size
176
+ and input_size_per_partition % block_k != 0):
177
+ raise ValueError(
178
+ f"Weight input_size_per_partition = "
179
+ f"{input_size_per_partition} is not divisible by "
180
+ f"weight quantization block_k = {block_k}.")
181
+ # Required by column parallel or enabling merged weights
182
+ if (tp_size > 1 and output_size // output_size_per_partition
183
+ == tp_size) or len(output_partition_sizes) > 1:
184
+ for output_partition_size in output_partition_sizes:
185
+ if output_partition_size % block_n != 0:
186
+ raise ValueError(
187
+ f"Weight output_partition_size = "
188
+ f"{output_partition_size} is not divisible by "
189
+ f"weight quantization block_n = {block_n}.")
190
+
191
layer.logical_widths = output_partition_sizes
192
193
layer.input_size_per_partition = input_size_per_partition
@@ -161,12 +212,29 @@ class Fp8LinearMethod(LinearMethodBase):
212
# Otherwise, wait until process_weights_after_loading.
213
if self.quant_config.is_checkpoint_fp8_serialized:
214
# WEIGHT SCALE
164
- scale = PerTensorScaleParameter(data=torch.empty(
165
- len(output_partition_sizes), dtype=torch.float32),
166
- weight_loader=weight_loader)
167
-
168
- scale[:] = torch.finfo(torch.float32).min
169
- layer.register_parameter("weight_scale", scale)
215
+ if not self.block_quant:
216
+ scale = PerTensorScaleParameter(
217
+ data=torch.empty(len(output_partition_sizes),
218
+ dtype=torch.float32),
219
+ weight_loader=weight_loader,
220
+ )
221
+ scale[:] = torch.finfo(torch.float32).min
222
+ layer.register_parameter("weight_scale", scale)
223
+ else:
224
+ assert self.quant_config.activation_scheme == "dynamic"
225
+ scale = BlockQuantScaleParameter(
226
+ data=torch.empty(
227
+ (output_size_per_partition + block_n - 1) // block_n,
228
+ (input_size_per_partition + block_k - 1) // block_k,
229
+ dtype=torch.float32,
230
+ ),
231
+ input_dim=1,
232
+ output_dim=0,
233
+ weight_loader=weight_loader,
234
+ )
235
+ scale[:] = torch.finfo(torch.float32).min
236
+ # The weight_scale_inv name is intentional for deepseekv3
237
+ layer.register_parameter("weight_scale_inv", scale)
238
239
# INPUT ACTIVATION SCALE
240
if self.quant_config.activation_scheme == "static":
@@ -180,6 +248,9 @@ class Fp8LinearMethod(LinearMethodBase):
248
layer.register_parameter("input_scale", None)
249
250
def process_weights_after_loading(self, layer: Module) -> None:
251
+ # Block quant doesn't need to process weights after loading
252
+ if self.block_quant:
253
+ return
254
layer.weight = torch.nn.Parameter(layer.weight.data,
255
requires_grad=False)
256
# If checkpoint not serialized fp8, quantize the weights.
@@ -266,6 +337,17 @@ class Fp8LinearMethod(LinearMethodBase):
337
size_k=layer.input_size_per_partition,
338
bias=bias)
339
340
+ if self.block_quant:
341
+ assert self.quant_config.weight_block_size is not None
342
+ return apply_w8a8_block_fp8_linear(
343
+ input=x,
344
+ weight=layer.weight,
345
+ block_size=self.quant_config.weight_block_size,
346
+ weight_scale=layer.weight_scale_inv,
347
+ input_scale=layer.input_scale,
348
+ bias=bias,
349
+ )
350
+
351
return apply_fp8_linear(
352
input=x,
353
weight=layer.weight,
@@ -291,6 +373,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
373
374
def __init__(self, quant_config: Fp8Config):
375
self.quant_config = quant_config
376
+ self.block_quant = self.quant_config.weight_block_size is not None
377
378
def create_weights(self, layer: Module, num_experts: int, hidden_size: int,
379
intermediate_size: int, params_dtype: torch.dtype,
@@ -298,6 +381,27 @@ class Fp8MoEMethod(FusedMoEMethodBase):
381
382
if self.quant_config.is_checkpoint_fp8_serialized:
383
params_dtype = torch.float8_e4m3fn
384
+ if self.block_quant:
385
+ assert self.quant_config.weight_block_size is not None
386
+ tp_size = get_tensor_model_parallel_world_size()
387
+ block_n, block_k = (
388
+ self.quant_config.weight_block_size[0],
389
+ self.quant_config.weight_block_size[1],
390
+ )
391
+ # NOTE: To ensure proper alignment of the block-wise quantization
392
+ # scales, the output_size of the weights for both the gate and up
393
+ # layers must be divisible by block_n.
394
+ # Required by column parallel or enabling merged weights
395
+ if intermediate_size % block_n != 0:
396
+ raise ValueError(
397
+ f"The output_size of gate's and up's weight = "
398
+ f"{intermediate_size} is not divisible by "
399
+ f"weight quantization block_n = {block_n}.")
400
+ if (tp_size > 1 and intermediate_size % block_k != 0):
401
+ # Required by row parallel
402
+ raise ValueError(f"The input_size of down's weight = "
403
+ f"{intermediate_size} is not divisible by "
404
+ f"weight quantization block_k = {block_k}.")
405
406
# WEIGHTS
407
w13_weight = torch.nn.Parameter(torch.empty(num_experts,
@@ -317,21 +421,45 @@ class Fp8MoEMethod(FusedMoEMethodBase):
421
set_weight_attrs(w2_weight, extra_weight_attrs)
422
423
# WEIGHT_SCALES
320
- # Allocate 2 scales for w1 and w3 respectively.
321
- # They will be combined to a single scale after weight loading.
322
- w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
323
- 2,
324
- dtype=torch.float32),
325
- requires_grad=False)
326
- layer.register_parameter("w13_weight_scale", w13_weight_scale)
424
+ if not self.block_quant:
425
+ # Allocate 2 scales for w1 and w3 respectively.
426
+ # They will be combined to a single scale after weight loading.
427
+ w13_weight_scale = torch.nn.Parameter(torch.ones(
428
+ num_experts, 2, dtype=torch.float32),
429
+ requires_grad=False)
430
+ w2_weight_scale = torch.nn.Parameter(torch.ones(
431
+ num_experts, dtype=torch.float32),
432
+ requires_grad=False)
433
+ layer.register_parameter("w13_weight_scale", w13_weight_scale)
434
+ layer.register_parameter("w2_weight_scale", w2_weight_scale)
435
+ else:
436
+ w13_weight_scale = torch.nn.Parameter(
437
+ torch.ones(
438
+ num_experts,
439
+ 2 * ((intermediate_size + block_n - 1) // block_n),
440
+ (hidden_size + block_k - 1) // block_k,
441
+ dtype=torch.float32,
442
+ ),
443
+ requires_grad=False,
444
+ )
445
+ w2_weight_scale = torch.nn.Parameter(
446
+ torch.ones(
447
+ num_experts,
448
+ (hidden_size + block_n - 1) // block_n,
449
+ (intermediate_size + block_k - 1) // block_k,
450
+ dtype=torch.float32,
451
+ ),
452
+ requires_grad=False,
453
+ )
454
+ layer.register_parameter("w13_weight_scale_inv", w13_weight_scale)
455
+ layer.register_parameter("w2_weight_scale_inv", w2_weight_scale)
456
+ assert self.quant_config.activation_scheme == "dynamic"
457
328
- w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
329
- dtype=torch.float32),
330
- requires_grad=False)
331
- layer.register_parameter("w2_weight_scale", w2_weight_scale)
458
# Add the quantization method used (per tensor/grouped/channel)
459
# to ensure the weight scales are loaded in properly
460
extra_weight_attrs.update(
461
+ {"quant_method": FusedMoeWeightScaleSupported.BLOCK.
462
+ value} if self.block_quant else
463
{"quant_method": FusedMoeWeightScaleSupported.TENSOR.value})
464
# If loading fp8 checkpoint, pass the weight loaders.
465
# If loading an fp16 checkpoint, do not (we will quantize in
@@ -364,7 +492,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
492
layer.w2_input_scale = None
493
494
def process_weights_after_loading(self, layer: Module) -> None:
367
-
495
+ # Block quant doesn't need to process weights after loading
496
+ if self.block_quant:
497
+ return
498
# If checkpoint is fp16, quantize in place.
499
if not self.quant_config.is_checkpoint_fp8_serialized:
500
# If rocm, use float8_e4m3fnuz as dtype
@@ -489,17 +619,22 @@ class Fp8MoEMethod(FusedMoEMethodBase):
619
num_expert_group=num_expert_group,
620
custom_routing_function=custom_routing_function)
621
492
- return fused_experts(x,
493
- layer.w13_weight,
494
- layer.w2_weight,
495
- topk_weights=topk_weights,
496
- topk_ids=topk_ids,
497
- inplace=True,
498
- use_fp8_w8a8=True,
499
- w1_scale=layer.w13_weight_scale,
500
- w2_scale=layer.w2_weight_scale,
501
- a1_scale=layer.w13_input_scale,
502
- a2_scale=layer.w2_input_scale)
622
+ return fused_experts(
623
+ x,
624
+ layer.w13_weight,
625
+ layer.w2_weight,
626
+ topk_weights=topk_weights,
627
+ topk_ids=topk_ids,
628
+ inplace=True,
629
+ use_fp8_w8a8=True,
630
+ w1_scale=(layer.w13_weight_scale_inv
631
+ if self.block_quant else layer.w13_weight_scale),
632
+ w2_scale=(layer.w2_weight_scale_inv
633
+ if self.block_quant else layer.w2_weight_scale),
634
+ a1_scale=layer.w13_input_scale,
635
+ a2_scale=layer.w2_input_scale,
636
+ block_shape=self.quant_config.weight_block_size,
637
+ )
638
639
640
class Fp8KVCacheMethod(BaseKVCacheMethod):
vllm/model_executor/layers/quantization/utils/fp8_utils.py
new
+353
@@ -0,0 +1,353 @@
1
+# Adapted from https://github.com/sgl-project/sglang/pull/2575
2
+from typing import List, Optional, Tuple
3
+
4
+import torch
5
+import triton
6
+import triton.language as tl
7
+
8
+
9
+def apply_w8a8_block_fp8_linear(
10
+ input: torch.Tensor,
11
+ weight: torch.Tensor,
12
+ block_size: List[int],
13
+ weight_scale: torch.Tensor,
14
+ input_scale: Optional[torch.Tensor] = None,
15
+ bias: Optional[torch.Tensor] = None,
16
+) -> torch.Tensor:
17
+ assert input_scale is None
18
+ # View input as 2D matrix for fp8 methods
19
+ input_2d = input.view(-1, input.shape[-1])
20
+ output_shape = [*input.shape[:-1], weight.shape[0]]
21
+
22
+ q_input, x_scale = per_token_group_quant_fp8(input_2d, block_size[1])
23
+ output = w8a8_block_fp8_matmul(q_input,
24
+ weight,
25
+ x_scale,
26
+ weight_scale,
27
+ block_size,
28
+ output_dtype=input.dtype)
29
+
30
+ if bias is not None:
31
+ output = output + bias
32
+ return output.to(dtype=input.dtype).view(*output_shape)
33
+
34
+
35
+def input_to_float8(
36
+ x: torch.Tensor,
37
+ dtype: torch.dtype = torch.float8_e4m3fn
38
+) -> Tuple[torch.Tensor, torch.Tensor]:
39
+ """This function quantizes input values to float8 values "
40
+ "with tensor-wise quantization."""
41
+ finfo = torch.finfo(dtype)
42
+ min_val, max_val = x.aminmax()
43
+ amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12)
44
+ scale = finfo.max / amax
45
+ x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max)
46
+ return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal()
47
+
48
+
49
+def block_quant_to_tensor_quant(
50
+ x_q_block: torch.Tensor,
51
+ x_s: torch.Tensor,
52
+ block_size: List[int],
53
+) -> Tuple[torch.Tensor, torch.Tensor]:
54
+ """This function converts block-wise quantization to tensor-wise
55
+ quantization. The inputs are block-wise quantization tensor `x_q_block`,
56
+ block-wise quantization scale and the block size.
57
+ The outputs are tensor-wise quantization tensor and tensor-wise
58
+ quantization scale. Note only float8 is supported for now.
59
+ """
60
+ block_n, block_k = block_size[0], block_size[1]
61
+ n, k = x_q_block.shape
62
+ n_tiles = (n + block_n - 1) // block_n
63
+ k_tiles = (k + block_k - 1) // block_k
64
+ assert n_tiles == x_s.shape[0]
65
+ assert k_tiles == x_s.shape[1]
66
+
67
+ x_dq_block = x_q_block.to(torch.float32)
68
+
69
+ x_dq_block_tiles = [[
70
+ x_dq_block[j * block_n:min((j + 1) * block_n, n),
71
+ i * block_k:min((i + 1) * block_k, k), ]
72
+ for i in range(k_tiles)
73
+ ] for j in range(n_tiles)]
74
+
75
+ for i in range(k_tiles):
76
+ for j in range(n_tiles):
77
+ x_dq_block_tiles[j][i][:, :] = x_dq_block_tiles[j][i] * x_s[j][i]
78
+
79
+ x_q_tensor, scale = input_to_float8(x_dq_block, dtype=x_q_block.dtype)
80
+ return x_q_tensor, scale
81
+
82
+
83
+@triton.jit
84
+def _per_token_group_quant_fp8(
85
+ # Pointers to inputs and output
86
+ y_ptr,
87
+ y_q_ptr,
88
+ y_s_ptr,
89
+ # Stride of input
90
+ y_stride,
91
+ # Columns of input
92
+ N,
93
+ # Avoid to divide zero
94
+ eps,
95
+ # Information for float8
96
+ fp8_min,
97
+ fp8_max,
98
+ # Meta-parameters
99
+ BLOCK: tl.constexpr,
100
+):
101
+ """A Triton-accelerated function to perform per-token-group
102
+ quantization on a tensor.
103
+ This function converts the tensor values into float8 values.
104
+ """
105
+ # Map the program id to the row of X and Y it should compute.
106
+ g_id = tl.program_id(0)
107
+ y_ptr += g_id * y_stride
108
+ y_q_ptr += g_id * y_stride
109
+ y_s_ptr += g_id
110
+
111
+ cols = tl.arange(0, BLOCK) # N <= BLOCK
112
+ mask = cols < N
113
+
114
+ y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32)
115
+ # Quant
116
+ _absmax = tl.maximum(tl.max(tl.abs(y)), eps)
117
+ y_s = _absmax / fp8_max
118
+ y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
119
+
120
+ tl.store(y_q_ptr + cols, y_q, mask=mask)
121
+ tl.store(y_s_ptr, y_s)
122
+
123
+
124
+def per_token_group_quant_fp8(
125
+ x: torch.Tensor,
126
+ group_size: int,
127
+ eps: float = 1e-10,
128
+ dtype: torch.dtype = torch.float8_e4m3fn,
129
+) -> Tuple[torch.Tensor, torch.Tensor]:
130
+ """Function to perform per-token-group quantization on an input tensor `x`.
131
+ It converts the tensor values into signed float8 values and returns the
132
+ quantized tensor along with the scaling factor used for quantization.
133
+ Args:
134
+ x: The input tenosr with ndim >= 2.
135
+ group_size: The group size used for quantization.
136
+ eps: The minimum to avoid dividing zero.
137
+ dtype: The dype of output tensor. Note that only `torch.float8_e4m3fn`
138
+ is supported for now.
139
+ Returns:
140
+ Tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
141
+ scaling factor for quantization.
142
+ """
143
+ assert (x.shape[-1] % group_size == 0), (
144
+ f"the last dimension of `x` {x.shape[-1]} must be divisible "
145
+ f"by `group_size` {group_size}")
146
+ assert x.is_contiguous(), "`x` must be contiguous"
147
+
148
+ finfo = torch.finfo(dtype)
149
+ fp8_min = finfo.min
150
+ fp8_max = finfo.max
151
+
152
+ x_q = torch.empty_like(x, device=x.device, dtype=dtype)
153
+ M = x.numel() // group_size
154
+ N = group_size
155
+ x_s = torch.empty(
156
+ x.shape[:-1] + (x.shape[-1] // group_size, ),
157
+ device=x.device,
158
+ dtype=torch.float32,
159
+ )
160
+
161
+ BLOCK = triton.next_power_of_2(N)
162
+ # heuristics for number of warps
163
+ num_warps = min(max(BLOCK // 256, 1), 8)
164
+ num_stages = 1
165
+ _per_token_group_quant_fp8[(M, )](
166
+ x,
167
+ x_q,
168
+ x_s,
169
+ group_size,
170
+ N,
171
+ eps,
172
+ fp8_min=fp8_min,
173
+ fp8_max=fp8_max,
174
+ BLOCK=BLOCK,
175
+ num_warps=num_warps,
176
+ num_stages=num_stages,
177
+ )
178
+
179
+ return x_q, x_s
180
+
181
+
182
+@triton.jit
183
+def _w8a8_block_fp8_matmul(
184
+ # Pointers to inputs and output
185
+ A,
186
+ B,
187
+ C,
188
+ As,
189
+ Bs,
190
+ # Shape for matmul
191
+ M,
192
+ N,
193
+ K,
194
+ # Block size for block-wise quantization
195
+ group_n,
196
+ group_k,
197
+ # Stride for inputs and output
198
+ stride_am,
199
+ stride_ak,
200
+ stride_bk,
201
+ stride_bn,
202
+ stride_cm,
203
+ stride_cn,
204
+ stride_As_m,
205
+ stride_As_k,
206
+ stride_Bs_k,
207
+ stride_Bs_n,
208
+ # Meta-parameters
209
+ BLOCK_SIZE_M: tl.constexpr,
210
+ BLOCK_SIZE_N: tl.constexpr,
211
+ BLOCK_SIZE_K: tl.constexpr,
212
+ GROUP_SIZE_M: tl.constexpr,
213
+):
214
+ """Triton-accelerated function used to perform linear operations (dot
215
+ product) on input tensors `A` and `B` with block-wise quantization, and
216
+ store the result in output tensor `C`.
217
+ """
218
+
219
+ pid = tl.program_id(axis=0)
220
+ num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
221
+ num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
222
+ num_pid_in_group = GROUP_SIZE_M * num_pid_n
223
+ group_id = pid // num_pid_in_group
224
+ first_pid_m = group_id * GROUP_SIZE_M
225
+ group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
226
+ pid_m = first_pid_m + (pid % group_size_m)
227
+ pid_n = (pid % num_pid_in_group) // group_size_m
228
+
229
+ offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M
230
+ offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N
231
+ offs_k = tl.arange(0, BLOCK_SIZE_K)
232
+ a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)
233
+ b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
234
+
235
+ As_ptrs = As + offs_am * stride_As_m
236
+ offs_bsn = offs_bn // group_n
237
+ Bs_ptrs = Bs + offs_bsn * stride_Bs_n
238
+
239
+ accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
240
+ for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
241
+ a = tl.load(a_ptrs,
242
+ mask=offs_k[None, :] < K - k * BLOCK_SIZE_K,
243
+ other=0.0)
244
+ b = tl.load(b_ptrs,
245
+ mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
246
+ other=0.0)
247
+
248
+ k_start = k * BLOCK_SIZE_K
249
+ offs_ks = k_start // group_k
250
+ a_s = tl.load(As_ptrs + offs_ks * stride_As_k)
251
+ b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k)
252
+
253
+ accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :]
254
+ a_ptrs += BLOCK_SIZE_K * stride_ak
255
+ b_ptrs += BLOCK_SIZE_K * stride_bk
256
+
257
+ if C.dtype.element_ty == tl.bfloat16:
258
+ c = accumulator.to(tl.bfloat16)
259
+ elif C.dtype.element_ty == tl.float16:
260
+ c = accumulator.to(tl.float16)
261
+ else:
262
+ c = accumulator.to(tl.float32)
263
+
264
+ offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
265
+ offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
266
+ c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
267
+ c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
268
+ tl.store(c_ptrs, c, mask=c_mask)
269
+
270
+
271
+def w8a8_block_fp8_matmul(
272
+ A: torch.Tensor,
273
+ B: torch.Tensor,
274
+ As: torch.Tensor,
275
+ Bs: torch.Tensor,
276
+ block_size: List[int],
277
+ output_dtype: torch.dtype = torch.float16,
278
+) -> torch.Tensor:
279
+ """This function performs matrix multiplication with block-wise
280
+ quantization.
281
+ It takes two input tensors `A` and `B` with scales `As` and `Bs`.
282
+ The output is returned in the specified `output_dtype`.
283
+ Args:
284
+ A: The input tensor, e.g., activation.
285
+ B: The input tensor, e.g., weight.
286
+ As: The per-token-group quantization scale for `A`.
287
+ Bs: The per-block quantization scale for `B`.
288
+ block_size: The block size for per-block quantization. It should
289
+ be 2-dim, e.g., [128, 128].
290
+ output_dytpe: The dtype of the returned tensor.
291
+ Returns:
292
+ torch.Tensor: The result of matmul.
293
+ """
294
+ assert len(block_size) == 2
295
+ block_n, block_k = block_size[0], block_size[1]
296
+
297
+ assert A.shape[-1] == B.shape[-1]
298
+ assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous()
299
+ assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1]
300
+ M = A.numel() // A.shape[-1]
301
+
302
+ assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2
303
+ N, K = B.shape
304
+ assert triton.cdiv(N, block_n) == Bs.shape[0]
305
+ assert triton.cdiv(K, block_k) == Bs.shape[1]
306
+
307
+ C_shape = A.shape[:-1] + (N, )
308
+ C = A.new_empty(C_shape, dtype=output_dtype)
309
+
310
+ # TODO:
311
+ # BLOCK_SIZE_M, BLOCK_SIZE_K, BLOCK_SIZE_N can be optimized.
312
+ # BLOCK_SIZE_K must be divisible by block_k
313
+ # BLOCK_SIZE_N and BLOCK_SIZE_M has no requirements
314
+ BLOCK_SIZE_M = 128
315
+ if M < BLOCK_SIZE_M:
316
+ BLOCK_SIZE_M = triton.next_power_of_2(M)
317
+ BLOCK_SIZE_M = max(BLOCK_SIZE_M, 16)
318
+ BLOCK_SIZE_K = block_k
319
+ assert block_k % BLOCK_SIZE_K == 0
320
+ BLOCK_SIZE_N = block_n
321
+
322
+ def grid(META):
323
+ return (triton.cdiv(M, META["BLOCK_SIZE_M"]) *
324
+ triton.cdiv(N, META["BLOCK_SIZE_N"]), )
325
+
326
+ _w8a8_block_fp8_matmul[grid](
327
+ A,
328
+ B,
329
+ C,
330
+ As,
331
+ Bs,
332
+ M,
333
+ N,
334
+ K,
335
+ block_n,
336
+ block_k,
337
+ A.stride(-2),
338
+ A.stride(-1),
339
+ B.stride(1),
340
+ B.stride(0),
341
+ C.stride(-2),
342
+ C.stride(-1),
343
+ As.stride(-2),
344
+ As.stride(-1),
345
+ Bs.stride(1),
346
+ Bs.stride(0),
347
+ BLOCK_SIZE_M=BLOCK_SIZE_M,
348
+ BLOCK_SIZE_N=BLOCK_SIZE_N,
349
+ BLOCK_SIZE_K=BLOCK_SIZE_K,
350
+ GROUP_SIZE_M=8,
351
+ )
352
+
353
+ return C
vllm/model_executor/parameter.py
+9
@@ -328,6 +328,15 @@ class PackedvLLMParameter(ModelWeightParameter):
328
marlin_tile_size=self.marlin_tile_size)
329
330
331
+class BlockQuantScaleParameter(_ColumnvLLMParameter, RowvLLMParameter):
332
+ """
333
+ Parameter class for weight scales loaded for weights with
334
+ block-wise quantization. Uses both column and row parallelism.
335
+ """
336
+
337
+ pass
338
+
339
+
340
def permute_param_layout_(param: BasevLLMParameter, input_dim: int,
341
output_dim: int, **kwargs) -> BasevLLMParameter:
342
"""