@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2072924d14
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Model] [Quantization] Support deepseek_v3 w8a8 fp8 block-wise quantization (#11523)

Signed-off-by: mgoin <michael@neuralmagic.com> Signed-off-by: simon-mo <simon.mo@hey.com> Signed-off-by: simon-mo <xmo@berkeley.edu> Co-authored-by: simon-mo <simon.mo@hey.com> Co-authored-by: simon-mo <xmo@berkeley.edu> Co-authored-by: HandH1998 <1335248067@qq.com>

Michael Goin committed Dec 26, 2024 at 18:33 UTC 2072924d1480460d3b3578a4548c2bffe33fe1c3
8 files changed +931 -70
tests/kernels/test_block_fp8.py new
+265
@@ -0,0 +1,265 @@
1 +# Adapted from https://github.com/sgl-project/sglang/pull/2575
2 +import itertools
3 +
4 +import pytest
5 +import torch
6 +
7 +from vllm.model_executor.layers.activation import SiluAndMul
8 +from vllm.model_executor.layers.fused_moe import fused_moe
9 +from vllm.model_executor.layers.quantization.utils.fp8_utils import (
10 + per_token_group_quant_fp8, w8a8_block_fp8_matmul)
11 +from vllm.platforms import current_platform
12 +
13 +if current_platform.get_device_capability() < (9, 0):
14 + pytest.skip("FP8 Triton requires CUDA 9.0 or higher",
15 + allow_module_level=True)
16 +
17 +# Test configurations
18 +DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
19 +NUM_TOKENS = [7, 83, 2048]
20 +D = [512, 4096, 5120, 13824]
21 +GROUP_SIZE = [64, 128, 256, 512]
22 +M = [1, 7, 83, 512, 2048]
23 +N = [128, 512, 1024, 4096, 7748, 13824]
24 +K = [256, 4096, 5120, 3884, 13824]
25 +# Deepseek-V3's intermediate size 18432, so N is 18432*2/8=4608 at TP8
26 +# and its hidden size is 7168.
27 +M_moe = [1, 7, 83, 512, 2048]
28 +N_moe = [4608] # [128, 4608, 13824]
29 +K_moe = [7168] # [256, 7168, 13824]
30 +BLOCK_SIZE = [[128, 128]]
31 +E = [256] # [8, 24, 128, 256]
32 +TOP_KS = [1] # [1, 2, 6]
33 +OUT_DTYPES = [torch.bfloat16] # [torch.float32, torch.half, torch.bfloat16]
34 +SEEDS = [0]
35 +
36 +
37 +def native_per_token_group_quant_fp8(x,
38 + group_size,
39 + eps=1e-10,
40 + dtype=torch.float8_e4m3fn):
41 + """Function to perform per-token-group quantization on an input tensor
42 + `x` using native torch."""
43 + assert x.shape[-1] % group_size == 0, ("the last dimension of `x` cannot "
44 + "be divisible by `group_size`")
45 + assert x.is_contiguous(), "`x` is not contiguous"
46 +
47 + finfo = torch.finfo(dtype)
48 + fp8_min = finfo.min
49 + fp8_max = finfo.max
50 +
51 + x_ = x.reshape(x.numel() // group_size, group_size)
52 + amax = x_.abs().max(dim=-1,
53 + keepdim=True)[0].clamp(min=eps).to(torch.float32)
54 + x_s = amax / fp8_max
55 + x_q = (x_ / x_s).clamp(min=fp8_min, max=fp8_max).to(dtype)
56 + x_q = x_q.reshape(x.shape)
57 + x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size, ))
58 +
59 + return x_q, x_s
60 +
61 +
62 +def native_w8a8_block_fp8_matmul(A,
63 + B,
64 + As,
65 + Bs,
66 + block_size,
67 + output_dtype=torch.float16):
68 + """Matrix multiplication with block-wise quantization using native torch."""
69 + A = A.to(torch.float32)
70 + B = B.to(torch.float32)
71 + assert A.shape[-1] == B.shape[-1]
72 + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2
73 + assert len(block_size) == 2
74 + block_n, block_k = block_size[0], block_size[1]
75 + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1]
76 + assert A.shape[:-1] == As.shape[:-1]
77 +
78 + M = A.numel() // A.shape[-1]
79 + N, K = B.shape
80 + origin_C_shape = A.shape[:-1] + (N, )
81 + A = A.reshape(M, A.shape[-1])
82 + As = As.reshape(M, As.shape[-1])
83 + n_tiles = (N + block_n - 1) // block_n
84 + k_tiles = (K + block_k - 1) // block_k
85 + assert n_tiles == Bs.shape[0]
86 + assert k_tiles == Bs.shape[1]
87 +
88 + C_shape = (M, N)
89 + C = torch.zeros(C_shape, dtype=torch.float32, device=A.device)
90 +
91 + A_tiles = [
92 + A[:, i * block_k:min((i + 1) * block_k, K)] for i in range(k_tiles)
93 + ]
94 + B_tiles = [[
95 + B[j * block_n:min((j + 1) * block_n, N),
96 + i * block_k:min((i + 1) * block_k, K), ] for i in range(k_tiles)
97 + ] for j in range(n_tiles)]
98 + C_tiles = [
99 + C[:, j * block_n:min((j + 1) * block_n, N)] for j in range(n_tiles)
100 + ]
101 + As_tiles = [As[:, i:i + 1] for i in range(k_tiles)]
102 +
103 + for i in range(k_tiles):
104 + for j in range(n_tiles):
105 + a = A_tiles[i]
106 + b = B_tiles[j][i]
107 + c = C_tiles[j]
108 + s = As_tiles[i] * Bs[j][i]
109 + c[:, :] += torch.matmul(a, b.t()) * s
110 +
111 + C = C.reshape(origin_C_shape).to(output_dtype)
112 + return C
113 +
114 +
115 +def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
116 + """Fused moe with block-wise quantization using native torch."""
117 + B, D = a.shape
118 + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
119 + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
120 + score = torch.softmax(score, dim=-1, dtype=torch.float32)
121 + topk_weight, topk_ids = torch.topk(score, topk)
122 + topk_weight = topk_weight.view(-1)
123 + topk_ids = topk_ids.view(-1)
124 +
125 + _, block_k = block_shape[0], block_shape[1]
126 + a_q, a_s = native_per_token_group_quant_fp8(a, block_k)
127 + a_q = a_q.to(torch.float32)
128 + for i in range(w1.shape[0]):
129 + mask = topk_ids == i
130 + if mask.sum():
131 + inter_out = native_w8a8_block_fp8_matmul(a_q[mask],
132 + w1[i],
133 + a_s[mask],
134 + w1_s[i],
135 + block_shape,
136 + output_dtype=a.dtype)
137 + act_out = SiluAndMul().forward_native(inter_out)
138 + act_out_q, act_out_s = native_per_token_group_quant_fp8(
139 + act_out, block_k)
140 + act_out = act_out.to(torch.float32)
141 + out[mask] = native_w8a8_block_fp8_matmul(act_out_q,
142 + w2[i],
143 + act_out_s,
144 + w2_s[i],
145 + block_shape,
146 + output_dtype=a.dtype)
147 + return (out.view(B, -1, w2.shape[1]) *
148 + topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
149 +
150 +
151 +# Skip all tests if CUDA is not available
152 +pytest.importorskip("torch.cuda")
153 +
154 +
155 +@pytest.fixture(autouse=True)
156 +def setup_cuda():
157 + torch.set_default_device("cuda")
158 +
159 +
160 +@pytest.mark.parametrize("num_tokens,d,dtype,group_size,seed",
161 + itertools.product(NUM_TOKENS, D, DTYPES, GROUP_SIZE,
162 + SEEDS))
163 +@torch.inference_mode()
164 +def test_per_token_group_quant_fp8(num_tokens, d, dtype, group_size, seed):
165 + torch.manual_seed(seed)
166 + x = torch.rand(num_tokens, d, dtype=dtype)
167 +
168 + ref_out, ref_scale = native_per_token_group_quant_fp8(x, group_size)
169 + out, scale = per_token_group_quant_fp8(x, group_size)
170 +
171 + assert torch.allclose(out.to(torch.float32),
172 + ref_out.to(torch.float32),
173 + rtol=0.15)
174 + assert torch.allclose(scale, ref_scale)
175 +
176 +
177 +@pytest.mark.parametrize("M,N,K,block_size,out_dtype,seed",
178 + itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES,
179 + SEEDS))
180 +@torch.inference_mode()
181 +def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed):
182 + torch.manual_seed(seed)
183 + factor_for_scale = 1e-2
184 + fp8_info = torch.finfo(torch.float8_e4m3fn)
185 + fp8_max, fp8_min = fp8_info.max, fp8_info.min
186 +
187 + A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
188 + A_fp8 = A_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
189 +
190 + B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
191 + B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
192 +
193 + block_n, block_k = block_size[0], block_size[1]
194 + n_tiles = (N + block_n - 1) // block_n
195 + k_tiles = (K + block_k - 1) // block_k
196 +
197 + As = torch.rand(M, k_tiles, dtype=torch.float32) * factor_for_scale
198 + Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
199 +
200 + ref_out = native_w8a8_block_fp8_matmul(A_fp8, B_fp8, As, Bs, block_size,
201 + out_dtype)
202 + out = w8a8_block_fp8_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
203 +
204 + rel_diff = (torch.mean(
205 + torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) /
206 + torch.mean(torch.abs(ref_out.to(torch.float32))))
207 + assert rel_diff < 0.001
208 +
209 +
210 +@pytest.mark.parametrize("M,N,K,E,topk,block_size,dtype,seed",
211 + itertools.product(M_moe, N_moe, K_moe, E, TOP_KS,
212 + BLOCK_SIZE, DTYPES, SEEDS))
213 +@torch.inference_mode()
214 +def test_w8a8_block_fp8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
215 + torch.manual_seed(seed)
216 + factor_for_scale = 1e-2
217 + fp8_info = torch.finfo(torch.float8_e4m3fn)
218 + fp8_max, fp8_min = fp8_info.max, fp8_info.min
219 +
220 + a = torch.randn((M, K), dtype=dtype) / 10
221 +
222 + w1_bf16 = (torch.rand(
223 + (E, 2 * N, K), dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
224 + w1 = w1_bf16.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
225 + del w1_bf16
226 +
227 + w2_bf16 = (torch.rand((E, K, N), dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
228 + w2 = w2_bf16.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
229 + del w2_bf16
230 +
231 + block_n, block_k = block_size[0], block_size[1]
232 + n_tiles_w1 = (2 * N + block_n - 1) // block_n
233 + n_tiles_w2 = (K + block_n - 1) // block_n
234 + k_tiles_w1 = (K + block_k - 1) // block_k
235 + k_tiles_w2 = (N + block_k - 1) // block_k
236 +
237 + w1_s = torch.rand(
238 + (E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) * factor_for_scale
239 + w2_s = torch.rand(
240 + (E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) * factor_for_scale
241 +
242 + score = torch.randn((M, E), dtype=dtype)
243 +
244 + out = fused_moe(
245 + a,
246 + w1,
247 + w2,
248 + score,
249 + topk,
250 + renormalize=False,
251 + use_fp8_w8a8=True,
252 + w1_scale=w1_s,
253 + w2_scale=w2_s,
254 + block_shape=block_size,
255 + )
256 + ref_out = torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, score, topk,
257 + block_size)
258 +
259 + print(f"{out.sum()=}")
260 + print(f"{ref_out.sum()=}")
261 +
262 + rel_diff = (torch.mean(
263 + torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) /
264 + torch.mean(torch.abs(ref_out.to(torch.float32))))
265 + assert rel_diff < 0.03
vllm/config.py
+7 -7
@@ -161,7 +161,7 @@ class ModelConfig:
161 override default pooling config for the pooling model.
162 logits_processor_pattern: Optional regex pattern specifying valid
163 logits processor qualified names that can be passed with the
164 - `logits_processors` extra completion argument. Defaults to None,
164 + `logits_processors` extra completion argument. Defaults to None,
165 which allows no processors.
166 generation_config: Configuration parameter file for generation.
167 """
@@ -364,7 +364,7 @@ class ModelConfig:
364 def maybe_pull_model_tokenizer_for_s3(self, model: str,
365 tokenizer: str) -> None:
366 """
367 - Pull the model config or tokenizer to a temporary
367 + Pull the model config or tokenizer to a temporary
368 directory in case of S3.
369
370 Args:
@@ -866,14 +866,14 @@ class ModelConfig:
866
867 def get_diff_sampling_param(self) -> Dict[str, Any]:
868 """
869 - This method returns a dictionary containing the parameters
870 - that differ from the default sampling parameters, but only
871 - if `generation_config` is set. If `generation_config` is not
869 + This method returns a dictionary containing the parameters
870 + that differ from the default sampling parameters, but only
871 + if `generation_config` is set. If `generation_config` is not
872 set, an empty dictionary is returned.
873
874 Returns:
875 - Dict[str, Any]: A dictionary with the differing sampling
876 - parameters if `generation_config` is set, otherwise an
875 + Dict[str, Any]: A dictionary with the differing sampling
876 + parameters if `generation_config` is set, otherwise an
877 empty dictionary.
878 """
879 if self.generation_config is None:
vllm/model_executor/layers/fused_moe/fused_moe.py
+103 -28
@@ -2,7 +2,7 @@
2 import functools
3 import json
4 import os
5 -from typing import Any, Callable, Dict, Optional, Tuple
5 +from typing import Any, Callable, Dict, List, Optional, Tuple
6
7 import torch
8 import triton
@@ -11,6 +11,8 @@ import triton.language as tl
11 import vllm.envs as envs
12 from vllm import _custom_ops as ops
13 from vllm.logger import init_logger
14 +from vllm.model_executor.layers.quantization.utils.fp8_utils import (
15 + per_token_group_quant_fp8)
16 from vllm.platforms import current_platform
17 from vllm.utils import direct_register_custom_op
18
@@ -45,8 +47,14 @@ def fused_moe_kernel(
47 stride_bn,
48 stride_cm,
49 stride_cn,
50 + stride_asm,
51 + stride_ask,
52 stride_bse,
53 + stride_bsk,
54 stride_bsn,
55 + # Block size for block-wise quantization
56 + group_n: tl.constexpr,
57 + group_k: tl.constexpr,
58 # Meta-parameters
59 BLOCK_SIZE_M: tl.constexpr,
60 BLOCK_SIZE_N: tl.constexpr,
@@ -125,8 +133,14 @@ def fused_moe_kernel(
133 b_scale = tl.load(b_scale_ptrs)
134
135 if use_fp8_w8a8:
128 - a_scale = tl.load(a_scale_ptr)
129 - b_scale = tl.load(b_scale_ptr + off_experts)
136 + if group_k > 0 and group_n > 0:
137 + a_scale_ptrs = a_scale_ptr + (offs_token // top_k) * stride_asm
138 + offs_bsn = offs_bn // group_n
139 + b_scale_ptrs = (b_scale_ptr + off_experts * stride_bse +
140 + offs_bsn * stride_bsn)
141 + else:
142 + a_scale = tl.load(a_scale_ptr)
143 + b_scale = tl.load(b_scale_ptr + off_experts)
144
145 # -----------------------------------------------------------
146 # Iterate to compute a block of the C matrix.
@@ -149,7 +163,18 @@ def fused_moe_kernel(
163 if use_int8_w8a16:
164 accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
165 elif use_fp8_w8a8:
152 - accumulator = tl.dot(a, b, acc=accumulator)
166 + if group_k > 0 and group_n > 0:
167 + k_start = k * BLOCK_SIZE_K
168 + offs_ks = k_start // group_k
169 + a_scale = tl.load(a_scale_ptrs + offs_ks * stride_ask,
170 + mask=token_mask,
171 + other=0.0)
172 + b_scale = tl.load(b_scale_ptrs + offs_ks * stride_bsk)
173 +
174 + accumulator += tl.dot(a, b) * a_scale[:,
175 + None] * b_scale[None, :]
176 + else:
177 + accumulator = tl.dot(a, b, acc=accumulator)
178 else:
179 accumulator += tl.dot(a, b)
180 # Advance the ptrs to the next K block.
@@ -164,7 +189,10 @@ def fused_moe_kernel(
189 if use_int8_w8a16:
190 accumulator = (accumulator * b_scale).to(compute_type)
191 elif use_fp8_w8a8:
167 - accumulator = (accumulator * a_scale * b_scale).to(compute_type)
192 + if group_k > 0 and group_n > 0:
193 + accumulator = accumulator.to(compute_type)
194 + else:
195 + accumulator = (accumulator * a_scale * b_scale).to(compute_type)
196 else:
197 accumulator = accumulator.to(compute_type)
198 # -----------------------------------------------------------
@@ -233,22 +261,37 @@ def moe_align_block_size(
261 return sorted_ids, expert_ids, num_tokens_post_pad
262
263
236 -def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
264 +def invoke_fused_moe_kernel(A: torch.Tensor,
265 + B: torch.Tensor,
266 + C: torch.Tensor,
267 A_scale: Optional[torch.Tensor],
268 B_scale: Optional[torch.Tensor],
239 - topk_weights: torch.Tensor, topk_ids: torch.Tensor,
269 + topk_weights: torch.Tensor,
270 + topk_ids: torch.Tensor,
271 sorted_token_ids: torch.Tensor,
272 expert_ids: torch.Tensor,
273 num_tokens_post_padded: torch.Tensor,
243 - mul_routed_weight: bool, top_k: int,
244 - config: Dict[str, Any], compute_type: tl.dtype,
245 - use_fp8_w8a8: bool, use_int8_w8a16: bool) -> None:
274 + mul_routed_weight: bool,
275 + top_k: int,
276 + config: Dict[str, Any],
277 + compute_type: tl.dtype,
278 + use_fp8_w8a8: bool,
279 + use_int8_w8a16: bool,
280 + block_shape: Optional[List[int]] = None) -> None:
281 assert topk_weights.stride(1) == 1
282 assert sorted_token_ids.stride(0) == 1
283
284 if use_fp8_w8a8:
250 - A, A_scale = ops.scaled_fp8_quant(A, A_scale)
285 assert B_scale is not None
286 + if block_shape is None:
287 + A, A_scale = ops.scaled_fp8_quant(A, A_scale)
288 + else:
289 + assert len(block_shape) == 2
290 + block_n, block_k = block_shape[0], block_shape[1]
291 + A, A_scale = per_token_group_quant_fp8(A, block_k)
292 + assert triton.cdiv(A.shape[-1], block_k) == A_scale.shape[-1]
293 + assert triton.cdiv(B.shape[-2], block_n) == B_scale.shape[-2]
294 + assert triton.cdiv(B.shape[-1], block_k) == B_scale.shape[-1]
295 elif use_int8_w8a16:
296 assert B_scale is not None
297 else:
@@ -279,8 +322,13 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
322 B.stride(1),
323 C.stride(1),
324 C.stride(2),
282 - B_scale.stride(0) if B_scale is not None and use_int8_w8a16 else 0,
283 - B_scale.stride(1) if B_scale is not None and use_int8_w8a16 else 0,
325 + A_scale.stride(0) if A_scale is not None and A_scale.ndim == 2 else 0,
326 + A_scale.stride(1) if A_scale is not None and A_scale.ndim == 2 else 0,
327 + B_scale.stride(0) if B_scale is not None and B_scale.ndim >= 2 else 0,
328 + B_scale.stride(2) if B_scale is not None and B_scale.ndim == 3 else 0,
329 + B_scale.stride(1) if B_scale is not None and B_scale.ndim >= 2 else 0,
330 + 0 if block_shape is None else block_shape[0],
331 + 0 if block_shape is None else block_shape[1],
332 MUL_ROUTED_WEIGHT=mul_routed_weight,
333 top_k=top_k,
334 compute_type=compute_type,
@@ -362,6 +410,7 @@ def try_get_optimal_moe_config(
410 dtype: Optional[str],
411 M: int,
412 is_marlin: bool = False,
413 + block_shape: Optional[List[int]] = None,
414 ):
415 from vllm.model_executor.layers.fused_moe import get_config
416 override_config = get_config()
@@ -380,6 +429,12 @@ def try_get_optimal_moe_config(
429 # Else use the default config
430 config = get_default_config(M, E, N, w1_shape[2], top_k, dtype,
431 is_marlin)
432 + # NOTE: For block-wise quant,
433 + # BLOCK_K must be divisible by block_shape[1]
434 + # BLOCK_N and BLOCK_M has no requirements
435 + if block_shape is not None:
436 + config["BLOCK_SIZE_N"] = block_shape[0]
437 + config["BLOCK_SIZE_K"] = block_shape[1]
438 return config
439
440
@@ -479,10 +534,11 @@ def inplace_fused_experts(hidden_states: torch.Tensor,
534 w1_scale: Optional[torch.Tensor] = None,
535 w2_scale: Optional[torch.Tensor] = None,
536 a1_scale: Optional[torch.Tensor] = None,
482 - a2_scale: Optional[torch.Tensor] = None) -> None:
537 + a2_scale: Optional[torch.Tensor] = None,
538 + block_shape: Optional[List[int]] = None) -> None:
539 fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids, True,
540 use_fp8_w8a8, use_int8_w8a16, w1_scale, w2_scale,
485 - a1_scale, a2_scale)
541 + a1_scale, a2_scale, block_shape)
542
543
544 def inplace_fused_experts_fake(
@@ -496,7 +552,8 @@ def inplace_fused_experts_fake(
552 w1_scale: Optional[torch.Tensor] = None,
553 w2_scale: Optional[torch.Tensor] = None,
554 a1_scale: Optional[torch.Tensor] = None,
499 - a2_scale: Optional[torch.Tensor] = None) -> None:
555 + a2_scale: Optional[torch.Tensor] = None,
556 + block_shape: Optional[List[int]] = None) -> None:
557 pass
558
559
@@ -519,10 +576,11 @@ def outplace_fused_experts(
576 w1_scale: Optional[torch.Tensor] = None,
577 w2_scale: Optional[torch.Tensor] = None,
578 a1_scale: Optional[torch.Tensor] = None,
522 - a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
579 + a2_scale: Optional[torch.Tensor] = None,
580 + block_shape: Optional[List[int]] = None) -> torch.Tensor:
581 return fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids,
582 False, use_fp8_w8a8, use_int8_w8a16, w1_scale,
525 - w2_scale, a1_scale, a2_scale)
583 + w2_scale, a1_scale, a2_scale, block_shape)
584
585
586 def outplace_fused_experts_fake(
@@ -536,7 +594,8 @@ def outplace_fused_experts_fake(
594 w1_scale: Optional[torch.Tensor] = None,
595 w2_scale: Optional[torch.Tensor] = None,
596 a1_scale: Optional[torch.Tensor] = None,
539 - a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
597 + a2_scale: Optional[torch.Tensor] = None,
598 + block_shape: Optional[List[int]] = None) -> torch.Tensor:
599 return torch.empty_like(hidden_states)
600
601
@@ -559,18 +618,22 @@ def fused_experts(hidden_states: torch.Tensor,
618 w1_scale: Optional[torch.Tensor] = None,
619 w2_scale: Optional[torch.Tensor] = None,
620 a1_scale: Optional[torch.Tensor] = None,
562 - a2_scale: Optional[torch.Tensor] = None):
621 + a2_scale: Optional[torch.Tensor] = None,
622 + block_shape: Optional[List[int]] = None):
623 if inplace:
624 torch.ops.vllm.inplace_fused_experts(hidden_states, w1, w2,
625 topk_weights, topk_ids,
626 use_fp8_w8a8, use_int8_w8a16,
627 w1_scale, w2_scale, a1_scale,
568 - a2_scale)
628 + a2_scale, block_shape)
629 return hidden_states
630 else:
571 - return torch.ops.vllm.outplace_fused_experts(
572 - hidden_states, w1, w2, topk_weights, topk_ids, use_fp8_w8a8,
573 - use_int8_w8a16, w1_scale, w2_scale, a1_scale, a2_scale)
631 + return torch.ops.vllm.outplace_fused_experts(hidden_states, w1, w2,
632 + topk_weights, topk_ids,
633 + use_fp8_w8a8,
634 + use_int8_w8a16, w1_scale,
635 + w2_scale, a1_scale,
636 + a2_scale, block_shape)
637
638
639 def fused_experts_impl(hidden_states: torch.Tensor,
@@ -584,7 +647,8 @@ def fused_experts_impl(hidden_states: torch.Tensor,
647 w1_scale: Optional[torch.Tensor] = None,
648 w2_scale: Optional[torch.Tensor] = None,
649 a1_scale: Optional[torch.Tensor] = None,
587 - a2_scale: Optional[torch.Tensor] = None):
650 + a2_scale: Optional[torch.Tensor] = None,
651 + block_shape: Optional[List[int]] = None):
652 # Check constraints.
653 assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
654 assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
@@ -611,6 +675,7 @@ def fused_experts_impl(hidden_states: torch.Tensor,
675 w2.shape,
676 topk_ids.shape[1],
677 config_dtype,
678 + block_shape=block_shape,
679 )
680
681 config = get_config_func(M)
@@ -674,7 +739,8 @@ def fused_experts_impl(hidden_states: torch.Tensor,
739 config,
740 compute_type=compute_type,
741 use_fp8_w8a8=use_fp8_w8a8,
677 - use_int8_w8a16=use_int8_w8a16)
742 + use_int8_w8a16=use_int8_w8a16,
743 + block_shape=block_shape)
744
745 ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
746
@@ -693,7 +759,8 @@ def fused_experts_impl(hidden_states: torch.Tensor,
759 config,
760 compute_type=compute_type,
761 use_fp8_w8a8=use_fp8_w8a8,
696 - use_int8_w8a16=use_int8_w8a16)
762 + use_int8_w8a16=use_int8_w8a16,
763 + block_shape=block_shape)
764
765 ops.moe_sum(intermediate_cache3.view(*intermediate_cache3.shape),
766 out_hidden_states[begin_chunk_idx:end_chunk_idx])
@@ -718,6 +785,7 @@ def fused_moe(
785 w2_scale: Optional[torch.Tensor] = None,
786 a1_scale: Optional[torch.Tensor] = None,
787 a2_scale: Optional[torch.Tensor] = None,
788 + block_shape: Optional[List[int]] = None,
789 ) -> torch.Tensor:
790 """
791 This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -745,6 +813,12 @@ def fused_moe(
813 w1.
814 - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
815 w2.
816 + - a1_scale (Optional[torch.Tensor]): Optional scale to be used for
817 + a1.
818 + - a2_scale (Optional[torch.Tensor]): Optional scale to be used for
819 + a2.
820 + - block_shape: (Optional[List[int]]): Optional block size for block-wise
821 + quantization.
822
823 Returns:
824 - torch.Tensor: The output tensor after applying the MoE layer.
@@ -775,4 +849,5 @@ def fused_moe(
849 w1_scale=w1_scale,
850 w2_scale=w2_scale,
851 a1_scale=a1_scale,
778 - a2_scale=a2_scale)
852 + a2_scale=a2_scale,
853 + block_shape=block_shape)
vllm/model_executor/layers/fused_moe/layer.py
+6 -1
@@ -29,6 +29,7 @@ class FusedMoeWeightScaleSupported(Enum):
29 TENSOR = "tensor"
30 CHANNEL = "channel"
31 GROUP = "group"
32 + BLOCK = "block"
33
34
35 class FusedMoEMethodBase(QuantizeMethodBase):
@@ -199,6 +200,7 @@ class FusedMoE(torch.nn.Module):
200 get_tensor_model_parallel_world_size())
201 self.top_k = top_k
202 self.num_experts = num_experts
203 + assert intermediate_size % self.tp_size == 0
204 self.intermediate_size_per_partition = intermediate_size // self.tp_size
205 self.reduce_results = reduce_results
206 self.renormalize = renormalize
@@ -398,7 +400,10 @@ class FusedMoE(torch.nn.Module):
400 loaded_weight=loaded_weight,
401 expert_data=expert_data,
402 tp_rank=tp_rank)
401 - elif quant_method == FusedMoeWeightScaleSupported.GROUP.value:
403 + elif quant_method in [
404 + FusedMoeWeightScaleSupported.GROUP.value,
405 + FusedMoeWeightScaleSupported.BLOCK.value,
406 + ]:
407 self._load_model_weight_or_group_weight_scale(
408 shard_id=shard_id,
409 shard_dim=shard_dim,
vllm/model_executor/layers/linear.py
+21 -2
@@ -14,11 +14,14 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
14 from vllm.logger import init_logger
15 from vllm.model_executor.layers.quantization.base_config import (
16 QuantizationConfig, QuantizeMethodBase)
17 +# yapf: disable
18 from vllm.model_executor.parameter import (BasevLLMParameter,
19 + BlockQuantScaleParameter,
20 PackedColumnParameter,
21 PackedvLLMParameter,
22 PerTensorScaleParameter,
23 RowvLLMParameter)
24 +# yapf: enable
25 from vllm.model_executor.utils import set_weight_attrs
26
27 logger = init_logger(__name__)
@@ -623,8 +626,24 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
626 assert loaded_shard_id < len(self.output_sizes)
627
628 tp_size = get_tensor_model_parallel_world_size()
626 - shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
627 - shard_size = self.output_sizes[loaded_shard_id] // tp_size
629 +
630 + if isinstance(param, BlockQuantScaleParameter):
631 + from vllm.model_executor.layers.quantization.fp8 import (
632 + Fp8LinearMethod, Fp8MoEMethod)
633 + assert self.quant_method is not None
634 + assert isinstance(self.quant_method,
635 + (Fp8LinearMethod, Fp8MoEMethod))
636 + weight_block_size = self.quant_method.quant_config.weight_block_size
637 + assert weight_block_size is not None
638 + block_n, _ = weight_block_size[0], weight_block_size[1]
639 + shard_offset = (
640 + (sum(self.output_sizes[:loaded_shard_id]) + block_n - 1) //
641 + block_n) // tp_size
642 + shard_size = ((self.output_sizes[loaded_shard_id] + block_n - 1) //
643 + block_n // tp_size)
644 + else:
645 + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
646 + shard_size = self.output_sizes[loaded_shard_id] // tp_size
647
648 param.load_merged_column_weight(loaded_weight=loaded_weight,
649 shard_id=loaded_shard_id,
vllm/model_executor/layers/quantization/fp8.py
+167 -32
@@ -6,6 +6,7 @@ from torch.nn.parameter import Parameter
6
7 import vllm.envs as envs
8 from vllm import _custom_ops as ops
9 +from vllm.distributed import get_tensor_model_parallel_world_size
10 from vllm.logger import init_logger
11 from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
12 FusedMoeWeightScaleSupported)
@@ -14,6 +15,8 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
15 from vllm.model_executor.layers.quantization.base_config import (
16 QuantizationConfig, QuantizeMethodBase)
17 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
18 +from vllm.model_executor.layers.quantization.utils.fp8_utils import (
19 + apply_w8a8_block_fp8_linear)
20 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
21 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
22 from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -22,7 +25,8 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
25 all_close_1d, apply_fp8_linear, convert_to_channelwise,
26 cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
27 requantize_with_max_scale)
25 -from vllm.model_executor.parameter import (ModelWeightParameter,
28 +from vllm.model_executor.parameter import (BlockQuantScaleParameter,
29 + ModelWeightParameter,
30 PerTensorScaleParameter)
31 from vllm.model_executor.utils import set_weight_attrs
32 from vllm.platforms import current_platform
@@ -41,6 +45,7 @@ class Fp8Config(QuantizationConfig):
45 is_checkpoint_fp8_serialized: bool = False,
46 activation_scheme: str = "dynamic",
47 ignored_layers: Optional[List[str]] = None,
48 + weight_block_size: Optional[List[int]] = None,
49 ) -> None:
50 self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized
51 if is_checkpoint_fp8_serialized:
@@ -51,6 +56,20 @@ class Fp8Config(QuantizationConfig):
56 f"Unsupported activation scheme {activation_scheme}")
57 self.activation_scheme = activation_scheme
58 self.ignored_layers = ignored_layers or []
59 + if weight_block_size is not None:
60 + if not is_checkpoint_fp8_serialized:
61 + raise ValueError(
62 + "The block-wise quantization only supports fp8-serialized "
63 + "checkpoint for now.")
64 + if len(weight_block_size) != 2:
65 + raise ValueError(
66 + "The quantization block size of weight must have 2 "
67 + f"dimensions, but got {len(weight_block_size)} dimensions")
68 + if activation_scheme != "dynamic":
69 + raise ValueError("The block-wise quantization only supports "
70 + "dynamic activation scheme for now, but got "
71 + f"{activation_scheme} activation scheme.")
72 + self.weight_block_size = weight_block_size
73
74 @classmethod
75 def get_name(cls) -> str:
@@ -74,9 +93,12 @@ class Fp8Config(QuantizationConfig):
93 is_checkpoint_fp8_serialized = ("fp8" in quant_method)
94 activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
95 ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
96 + weight_block_size = cls.get_from_keys_or(config, ["weight_block_size"],
97 + None)
98 return cls(is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
99 activation_scheme=activation_scheme,
79 - ignored_layers=ignored_layers)
100 + ignored_layers=ignored_layers,
101 + weight_block_size=weight_block_size)
102
103 def get_quant_method(self, layer: torch.nn.Module,
104 prefix: str) -> Optional["QuantizeMethodBase"]:
@@ -123,6 +145,11 @@ class Fp8LinearMethod(LinearMethodBase):
145 if current_platform.is_rocm():
146 self.use_marlin = False
147
148 + self.block_quant = self.quant_config.weight_block_size is not None
149 + if self.block_quant:
150 + # Marlin doesn't support block-wise fp8
151 + self.use_marlin = False
152 +
153 def create_weights(
154 self,
155 layer: torch.nn.Module,
@@ -133,10 +160,34 @@ class Fp8LinearMethod(LinearMethodBase):
160 params_dtype: torch.dtype,
161 **extra_weight_attrs,
162 ):
136 - del input_size, output_size
163 output_size_per_partition = sum(output_partition_sizes)
164 weight_loader = extra_weight_attrs.get("weight_loader")
165
166 + if self.block_quant:
167 + tp_size = get_tensor_model_parallel_world_size()
168 + assert self.quant_config.weight_block_size is not None
169 + block_n, block_k = (
170 + self.quant_config.weight_block_size[0],
171 + self.quant_config.weight_block_size[1],
172 + )
173 + # Required by row parallel
174 + if (tp_size > 1
175 + and input_size // input_size_per_partition == tp_size
176 + and input_size_per_partition % block_k != 0):
177 + raise ValueError(
178 + f"Weight input_size_per_partition = "
179 + f"{input_size_per_partition} is not divisible by "
180 + f"weight quantization block_k = {block_k}.")
181 + # Required by column parallel or enabling merged weights
182 + if (tp_size > 1 and output_size // output_size_per_partition
183 + == tp_size) or len(output_partition_sizes) > 1:
184 + for output_partition_size in output_partition_sizes:
185 + if output_partition_size % block_n != 0:
186 + raise ValueError(
187 + f"Weight output_partition_size = "
188 + f"{output_partition_size} is not divisible by "
189 + f"weight quantization block_n = {block_n}.")
190 +
191 layer.logical_widths = output_partition_sizes
192
193 layer.input_size_per_partition = input_size_per_partition
@@ -161,12 +212,29 @@ class Fp8LinearMethod(LinearMethodBase):
212 # Otherwise, wait until process_weights_after_loading.
213 if self.quant_config.is_checkpoint_fp8_serialized:
214 # WEIGHT SCALE
164 - scale = PerTensorScaleParameter(data=torch.empty(
165 - len(output_partition_sizes), dtype=torch.float32),
166 - weight_loader=weight_loader)
167 -
168 - scale[:] = torch.finfo(torch.float32).min
169 - layer.register_parameter("weight_scale", scale)
215 + if not self.block_quant:
216 + scale = PerTensorScaleParameter(
217 + data=torch.empty(len(output_partition_sizes),
218 + dtype=torch.float32),
219 + weight_loader=weight_loader,
220 + )
221 + scale[:] = torch.finfo(torch.float32).min
222 + layer.register_parameter("weight_scale", scale)
223 + else:
224 + assert self.quant_config.activation_scheme == "dynamic"
225 + scale = BlockQuantScaleParameter(
226 + data=torch.empty(
227 + (output_size_per_partition + block_n - 1) // block_n,
228 + (input_size_per_partition + block_k - 1) // block_k,
229 + dtype=torch.float32,
230 + ),
231 + input_dim=1,
232 + output_dim=0,
233 + weight_loader=weight_loader,
234 + )
235 + scale[:] = torch.finfo(torch.float32).min
236 + # The weight_scale_inv name is intentional for deepseekv3
237 + layer.register_parameter("weight_scale_inv", scale)
238
239 # INPUT ACTIVATION SCALE
240 if self.quant_config.activation_scheme == "static":
@@ -180,6 +248,9 @@ class Fp8LinearMethod(LinearMethodBase):
248 layer.register_parameter("input_scale", None)
249
250 def process_weights_after_loading(self, layer: Module) -> None:
251 + # Block quant doesn't need to process weights after loading
252 + if self.block_quant:
253 + return
254 layer.weight = torch.nn.Parameter(layer.weight.data,
255 requires_grad=False)
256 # If checkpoint not serialized fp8, quantize the weights.
@@ -266,6 +337,17 @@ class Fp8LinearMethod(LinearMethodBase):
337 size_k=layer.input_size_per_partition,
338 bias=bias)
339
340 + if self.block_quant:
341 + assert self.quant_config.weight_block_size is not None
342 + return apply_w8a8_block_fp8_linear(
343 + input=x,
344 + weight=layer.weight,
345 + block_size=self.quant_config.weight_block_size,
346 + weight_scale=layer.weight_scale_inv,
347 + input_scale=layer.input_scale,
348 + bias=bias,
349 + )
350 +
351 return apply_fp8_linear(
352 input=x,
353 weight=layer.weight,
@@ -291,6 +373,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
373
374 def __init__(self, quant_config: Fp8Config):
375 self.quant_config = quant_config
376 + self.block_quant = self.quant_config.weight_block_size is not None
377
378 def create_weights(self, layer: Module, num_experts: int, hidden_size: int,
379 intermediate_size: int, params_dtype: torch.dtype,
@@ -298,6 +381,27 @@ class Fp8MoEMethod(FusedMoEMethodBase):
381
382 if self.quant_config.is_checkpoint_fp8_serialized:
383 params_dtype = torch.float8_e4m3fn
384 + if self.block_quant:
385 + assert self.quant_config.weight_block_size is not None
386 + tp_size = get_tensor_model_parallel_world_size()
387 + block_n, block_k = (
388 + self.quant_config.weight_block_size[0],
389 + self.quant_config.weight_block_size[1],
390 + )
391 + # NOTE: To ensure proper alignment of the block-wise quantization
392 + # scales, the output_size of the weights for both the gate and up
393 + # layers must be divisible by block_n.
394 + # Required by column parallel or enabling merged weights
395 + if intermediate_size % block_n != 0:
396 + raise ValueError(
397 + f"The output_size of gate's and up's weight = "
398 + f"{intermediate_size} is not divisible by "
399 + f"weight quantization block_n = {block_n}.")
400 + if (tp_size > 1 and intermediate_size % block_k != 0):
401 + # Required by row parallel
402 + raise ValueError(f"The input_size of down's weight = "
403 + f"{intermediate_size} is not divisible by "
404 + f"weight quantization block_k = {block_k}.")
405
406 # WEIGHTS
407 w13_weight = torch.nn.Parameter(torch.empty(num_experts,
@@ -317,21 +421,45 @@ class Fp8MoEMethod(FusedMoEMethodBase):
421 set_weight_attrs(w2_weight, extra_weight_attrs)
422
423 # WEIGHT_SCALES
320 - # Allocate 2 scales for w1 and w3 respectively.
321 - # They will be combined to a single scale after weight loading.
322 - w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
323 - 2,
324 - dtype=torch.float32),
325 - requires_grad=False)
326 - layer.register_parameter("w13_weight_scale", w13_weight_scale)
424 + if not self.block_quant:
425 + # Allocate 2 scales for w1 and w3 respectively.
426 + # They will be combined to a single scale after weight loading.
427 + w13_weight_scale = torch.nn.Parameter(torch.ones(
428 + num_experts, 2, dtype=torch.float32),
429 + requires_grad=False)
430 + w2_weight_scale = torch.nn.Parameter(torch.ones(
431 + num_experts, dtype=torch.float32),
432 + requires_grad=False)
433 + layer.register_parameter("w13_weight_scale", w13_weight_scale)
434 + layer.register_parameter("w2_weight_scale", w2_weight_scale)
435 + else:
436 + w13_weight_scale = torch.nn.Parameter(
437 + torch.ones(
438 + num_experts,
439 + 2 * ((intermediate_size + block_n - 1) // block_n),
440 + (hidden_size + block_k - 1) // block_k,
441 + dtype=torch.float32,
442 + ),
443 + requires_grad=False,
444 + )
445 + w2_weight_scale = torch.nn.Parameter(
446 + torch.ones(
447 + num_experts,
448 + (hidden_size + block_n - 1) // block_n,
449 + (intermediate_size + block_k - 1) // block_k,
450 + dtype=torch.float32,
451 + ),
452 + requires_grad=False,
453 + )
454 + layer.register_parameter("w13_weight_scale_inv", w13_weight_scale)
455 + layer.register_parameter("w2_weight_scale_inv", w2_weight_scale)
456 + assert self.quant_config.activation_scheme == "dynamic"
457
328 - w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
329 - dtype=torch.float32),
330 - requires_grad=False)
331 - layer.register_parameter("w2_weight_scale", w2_weight_scale)
458 # Add the quantization method used (per tensor/grouped/channel)
459 # to ensure the weight scales are loaded in properly
460 extra_weight_attrs.update(
461 + {"quant_method": FusedMoeWeightScaleSupported.BLOCK.
462 + value} if self.block_quant else
463 {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value})
464 # If loading fp8 checkpoint, pass the weight loaders.
465 # If loading an fp16 checkpoint, do not (we will quantize in
@@ -364,7 +492,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
492 layer.w2_input_scale = None
493
494 def process_weights_after_loading(self, layer: Module) -> None:
367 -
495 + # Block quant doesn't need to process weights after loading
496 + if self.block_quant:
497 + return
498 # If checkpoint is fp16, quantize in place.
499 if not self.quant_config.is_checkpoint_fp8_serialized:
500 # If rocm, use float8_e4m3fnuz as dtype
@@ -489,17 +619,22 @@ class Fp8MoEMethod(FusedMoEMethodBase):
619 num_expert_group=num_expert_group,
620 custom_routing_function=custom_routing_function)
621
492 - return fused_experts(x,
493 - layer.w13_weight,
494 - layer.w2_weight,
495 - topk_weights=topk_weights,
496 - topk_ids=topk_ids,
497 - inplace=True,
498 - use_fp8_w8a8=True,
499 - w1_scale=layer.w13_weight_scale,
500 - w2_scale=layer.w2_weight_scale,
501 - a1_scale=layer.w13_input_scale,
502 - a2_scale=layer.w2_input_scale)
622 + return fused_experts(
623 + x,
624 + layer.w13_weight,
625 + layer.w2_weight,
626 + topk_weights=topk_weights,
627 + topk_ids=topk_ids,
628 + inplace=True,
629 + use_fp8_w8a8=True,
630 + w1_scale=(layer.w13_weight_scale_inv
631 + if self.block_quant else layer.w13_weight_scale),
632 + w2_scale=(layer.w2_weight_scale_inv
633 + if self.block_quant else layer.w2_weight_scale),
634 + a1_scale=layer.w13_input_scale,
635 + a2_scale=layer.w2_input_scale,
636 + block_shape=self.quant_config.weight_block_size,
637 + )
638
639
640 class Fp8KVCacheMethod(BaseKVCacheMethod):
vllm/model_executor/layers/quantization/utils/fp8_utils.py new
+353
@@ -0,0 +1,353 @@
1 +# Adapted from https://github.com/sgl-project/sglang/pull/2575
2 +from typing import List, Optional, Tuple
3 +
4 +import torch
5 +import triton
6 +import triton.language as tl
7 +
8 +
9 +def apply_w8a8_block_fp8_linear(
10 + input: torch.Tensor,
11 + weight: torch.Tensor,
12 + block_size: List[int],
13 + weight_scale: torch.Tensor,
14 + input_scale: Optional[torch.Tensor] = None,
15 + bias: Optional[torch.Tensor] = None,
16 +) -> torch.Tensor:
17 + assert input_scale is None
18 + # View input as 2D matrix for fp8 methods
19 + input_2d = input.view(-1, input.shape[-1])
20 + output_shape = [*input.shape[:-1], weight.shape[0]]
21 +
22 + q_input, x_scale = per_token_group_quant_fp8(input_2d, block_size[1])
23 + output = w8a8_block_fp8_matmul(q_input,
24 + weight,
25 + x_scale,
26 + weight_scale,
27 + block_size,
28 + output_dtype=input.dtype)
29 +
30 + if bias is not None:
31 + output = output + bias
32 + return output.to(dtype=input.dtype).view(*output_shape)
33 +
34 +
35 +def input_to_float8(
36 + x: torch.Tensor,
37 + dtype: torch.dtype = torch.float8_e4m3fn
38 +) -> Tuple[torch.Tensor, torch.Tensor]:
39 + """This function quantizes input values to float8 values "
40 + "with tensor-wise quantization."""
41 + finfo = torch.finfo(dtype)
42 + min_val, max_val = x.aminmax()
43 + amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12)
44 + scale = finfo.max / amax
45 + x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max)
46 + return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal()
47 +
48 +
49 +def block_quant_to_tensor_quant(
50 + x_q_block: torch.Tensor,
51 + x_s: torch.Tensor,
52 + block_size: List[int],
53 +) -> Tuple[torch.Tensor, torch.Tensor]:
54 + """This function converts block-wise quantization to tensor-wise
55 + quantization. The inputs are block-wise quantization tensor `x_q_block`,
56 + block-wise quantization scale and the block size.
57 + The outputs are tensor-wise quantization tensor and tensor-wise
58 + quantization scale. Note only float8 is supported for now.
59 + """
60 + block_n, block_k = block_size[0], block_size[1]
61 + n, k = x_q_block.shape
62 + n_tiles = (n + block_n - 1) // block_n
63 + k_tiles = (k + block_k - 1) // block_k
64 + assert n_tiles == x_s.shape[0]
65 + assert k_tiles == x_s.shape[1]
66 +
67 + x_dq_block = x_q_block.to(torch.float32)
68 +
69 + x_dq_block_tiles = [[
70 + x_dq_block[j * block_n:min((j + 1) * block_n, n),
71 + i * block_k:min((i + 1) * block_k, k), ]
72 + for i in range(k_tiles)
73 + ] for j in range(n_tiles)]
74 +
75 + for i in range(k_tiles):
76 + for j in range(n_tiles):
77 + x_dq_block_tiles[j][i][:, :] = x_dq_block_tiles[j][i] * x_s[j][i]
78 +
79 + x_q_tensor, scale = input_to_float8(x_dq_block, dtype=x_q_block.dtype)
80 + return x_q_tensor, scale
81 +
82 +
83 +@triton.jit
84 +def _per_token_group_quant_fp8(
85 + # Pointers to inputs and output
86 + y_ptr,
87 + y_q_ptr,
88 + y_s_ptr,
89 + # Stride of input
90 + y_stride,
91 + # Columns of input
92 + N,
93 + # Avoid to divide zero
94 + eps,
95 + # Information for float8
96 + fp8_min,
97 + fp8_max,
98 + # Meta-parameters
99 + BLOCK: tl.constexpr,
100 +):
101 + """A Triton-accelerated function to perform per-token-group
102 + quantization on a tensor.
103 + This function converts the tensor values into float8 values.
104 + """
105 + # Map the program id to the row of X and Y it should compute.
106 + g_id = tl.program_id(0)
107 + y_ptr += g_id * y_stride
108 + y_q_ptr += g_id * y_stride
109 + y_s_ptr += g_id
110 +
111 + cols = tl.arange(0, BLOCK) # N <= BLOCK
112 + mask = cols < N
113 +
114 + y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32)
115 + # Quant
116 + _absmax = tl.maximum(tl.max(tl.abs(y)), eps)
117 + y_s = _absmax / fp8_max
118 + y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
119 +
120 + tl.store(y_q_ptr + cols, y_q, mask=mask)
121 + tl.store(y_s_ptr, y_s)
122 +
123 +
124 +def per_token_group_quant_fp8(
125 + x: torch.Tensor,
126 + group_size: int,
127 + eps: float = 1e-10,
128 + dtype: torch.dtype = torch.float8_e4m3fn,
129 +) -> Tuple[torch.Tensor, torch.Tensor]:
130 + """Function to perform per-token-group quantization on an input tensor `x`.
131 + It converts the tensor values into signed float8 values and returns the
132 + quantized tensor along with the scaling factor used for quantization.
133 + Args:
134 + x: The input tenosr with ndim >= 2.
135 + group_size: The group size used for quantization.
136 + eps: The minimum to avoid dividing zero.
137 + dtype: The dype of output tensor. Note that only `torch.float8_e4m3fn`
138 + is supported for now.
139 + Returns:
140 + Tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
141 + scaling factor for quantization.
142 + """
143 + assert (x.shape[-1] % group_size == 0), (
144 + f"the last dimension of `x` {x.shape[-1]} must be divisible "
145 + f"by `group_size` {group_size}")
146 + assert x.is_contiguous(), "`x` must be contiguous"
147 +
148 + finfo = torch.finfo(dtype)
149 + fp8_min = finfo.min
150 + fp8_max = finfo.max
151 +
152 + x_q = torch.empty_like(x, device=x.device, dtype=dtype)
153 + M = x.numel() // group_size
154 + N = group_size
155 + x_s = torch.empty(
156 + x.shape[:-1] + (x.shape[-1] // group_size, ),
157 + device=x.device,
158 + dtype=torch.float32,
159 + )
160 +
161 + BLOCK = triton.next_power_of_2(N)
162 + # heuristics for number of warps
163 + num_warps = min(max(BLOCK // 256, 1), 8)
164 + num_stages = 1
165 + _per_token_group_quant_fp8[(M, )](
166 + x,
167 + x_q,
168 + x_s,
169 + group_size,
170 + N,
171 + eps,
172 + fp8_min=fp8_min,
173 + fp8_max=fp8_max,
174 + BLOCK=BLOCK,
175 + num_warps=num_warps,
176 + num_stages=num_stages,
177 + )
178 +
179 + return x_q, x_s
180 +
181 +
182 +@triton.jit
183 +def _w8a8_block_fp8_matmul(
184 + # Pointers to inputs and output
185 + A,
186 + B,
187 + C,
188 + As,
189 + Bs,
190 + # Shape for matmul
191 + M,
192 + N,
193 + K,
194 + # Block size for block-wise quantization
195 + group_n,
196 + group_k,
197 + # Stride for inputs and output
198 + stride_am,
199 + stride_ak,
200 + stride_bk,
201 + stride_bn,
202 + stride_cm,
203 + stride_cn,
204 + stride_As_m,
205 + stride_As_k,
206 + stride_Bs_k,
207 + stride_Bs_n,
208 + # Meta-parameters
209 + BLOCK_SIZE_M: tl.constexpr,
210 + BLOCK_SIZE_N: tl.constexpr,
211 + BLOCK_SIZE_K: tl.constexpr,
212 + GROUP_SIZE_M: tl.constexpr,
213 +):
214 + """Triton-accelerated function used to perform linear operations (dot
215 + product) on input tensors `A` and `B` with block-wise quantization, and
216 + store the result in output tensor `C`.
217 + """
218 +
219 + pid = tl.program_id(axis=0)
220 + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
221 + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
222 + num_pid_in_group = GROUP_SIZE_M * num_pid_n
223 + group_id = pid // num_pid_in_group
224 + first_pid_m = group_id * GROUP_SIZE_M
225 + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
226 + pid_m = first_pid_m + (pid % group_size_m)
227 + pid_n = (pid % num_pid_in_group) // group_size_m
228 +
229 + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M
230 + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N
231 + offs_k = tl.arange(0, BLOCK_SIZE_K)
232 + a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)
233 + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
234 +
235 + As_ptrs = As + offs_am * stride_As_m
236 + offs_bsn = offs_bn // group_n
237 + Bs_ptrs = Bs + offs_bsn * stride_Bs_n
238 +
239 + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
240 + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
241 + a = tl.load(a_ptrs,
242 + mask=offs_k[None, :] < K - k * BLOCK_SIZE_K,
243 + other=0.0)
244 + b = tl.load(b_ptrs,
245 + mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
246 + other=0.0)
247 +
248 + k_start = k * BLOCK_SIZE_K
249 + offs_ks = k_start // group_k
250 + a_s = tl.load(As_ptrs + offs_ks * stride_As_k)
251 + b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k)
252 +
253 + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :]
254 + a_ptrs += BLOCK_SIZE_K * stride_ak
255 + b_ptrs += BLOCK_SIZE_K * stride_bk
256 +
257 + if C.dtype.element_ty == tl.bfloat16:
258 + c = accumulator.to(tl.bfloat16)
259 + elif C.dtype.element_ty == tl.float16:
260 + c = accumulator.to(tl.float16)
261 + else:
262 + c = accumulator.to(tl.float32)
263 +
264 + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
265 + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
266 + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
267 + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
268 + tl.store(c_ptrs, c, mask=c_mask)
269 +
270 +
271 +def w8a8_block_fp8_matmul(
272 + A: torch.Tensor,
273 + B: torch.Tensor,
274 + As: torch.Tensor,
275 + Bs: torch.Tensor,
276 + block_size: List[int],
277 + output_dtype: torch.dtype = torch.float16,
278 +) -> torch.Tensor:
279 + """This function performs matrix multiplication with block-wise
280 + quantization.
281 + It takes two input tensors `A` and `B` with scales `As` and `Bs`.
282 + The output is returned in the specified `output_dtype`.
283 + Args:
284 + A: The input tensor, e.g., activation.
285 + B: The input tensor, e.g., weight.
286 + As: The per-token-group quantization scale for `A`.
287 + Bs: The per-block quantization scale for `B`.
288 + block_size: The block size for per-block quantization. It should
289 + be 2-dim, e.g., [128, 128].
290 + output_dytpe: The dtype of the returned tensor.
291 + Returns:
292 + torch.Tensor: The result of matmul.
293 + """
294 + assert len(block_size) == 2
295 + block_n, block_k = block_size[0], block_size[1]
296 +
297 + assert A.shape[-1] == B.shape[-1]
298 + assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous()
299 + assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1]
300 + M = A.numel() // A.shape[-1]
301 +
302 + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2
303 + N, K = B.shape
304 + assert triton.cdiv(N, block_n) == Bs.shape[0]
305 + assert triton.cdiv(K, block_k) == Bs.shape[1]
306 +
307 + C_shape = A.shape[:-1] + (N, )
308 + C = A.new_empty(C_shape, dtype=output_dtype)
309 +
310 + # TODO:
311 + # BLOCK_SIZE_M, BLOCK_SIZE_K, BLOCK_SIZE_N can be optimized.
312 + # BLOCK_SIZE_K must be divisible by block_k
313 + # BLOCK_SIZE_N and BLOCK_SIZE_M has no requirements
314 + BLOCK_SIZE_M = 128
315 + if M < BLOCK_SIZE_M:
316 + BLOCK_SIZE_M = triton.next_power_of_2(M)
317 + BLOCK_SIZE_M = max(BLOCK_SIZE_M, 16)
318 + BLOCK_SIZE_K = block_k
319 + assert block_k % BLOCK_SIZE_K == 0
320 + BLOCK_SIZE_N = block_n
321 +
322 + def grid(META):
323 + return (triton.cdiv(M, META["BLOCK_SIZE_M"]) *
324 + triton.cdiv(N, META["BLOCK_SIZE_N"]), )
325 +
326 + _w8a8_block_fp8_matmul[grid](
327 + A,
328 + B,
329 + C,
330 + As,
331 + Bs,
332 + M,
333 + N,
334 + K,
335 + block_n,
336 + block_k,
337 + A.stride(-2),
338 + A.stride(-1),
339 + B.stride(1),
340 + B.stride(0),
341 + C.stride(-2),
342 + C.stride(-1),
343 + As.stride(-2),
344 + As.stride(-1),
345 + Bs.stride(1),
346 + Bs.stride(0),
347 + BLOCK_SIZE_M=BLOCK_SIZE_M,
348 + BLOCK_SIZE_N=BLOCK_SIZE_N,
349 + BLOCK_SIZE_K=BLOCK_SIZE_K,
350 + GROUP_SIZE_M=8,
351 + )
352 +
353 + return C
vllm/model_executor/parameter.py
+9
@@ -328,6 +328,15 @@ class PackedvLLMParameter(ModelWeightParameter):
328 marlin_tile_size=self.marlin_tile_size)
329
330
331 +class BlockQuantScaleParameter(_ColumnvLLMParameter, RowvLLMParameter):
332 + """
333 + Parameter class for weight scales loaded for weights with
334 + block-wise quantization. Uses both column and row parallelism.
335 + """
336 +
337 + pass
338 +
339 +
340 def permute_param_layout_(param: BasevLLMParameter, input_dim: int,
341 output_dim: int, **kwargs) -> BasevLLMParameter:
342 """