A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Kernel] W8A16 Int8 inside FusedMoE (#7415)
Mor Zusman committed
Aug 16, 2024 at 20:06 UTC
7fc23be81c55ca0570f551871a3adc994aaefc05
15 files changed
+412
-136
benchmarks/kernels/benchmark_moe.py
+70
-38
@@ -30,19 +30,36 @@ def benchmark_config(
30
hidden_size: int,
31
topk: int,
32
dtype: torch.dtype,
33
- use_fp8: bool,
33
+ use_fp8_w8a8: bool,
34
+ use_int8_w8a16: bool,
35
num_iters: int = 100,
36
) -> float:
36
- init_dtype = torch.float16 if use_fp8 else dtype
37
+ init_dtype = torch.float16 if use_fp8_w8a8 else dtype
38
x = torch.randn(num_tokens, hidden_size, dtype=dtype)
38
- w1 = torch.randn(num_experts,
39
- shard_intermediate_size,
40
- hidden_size,
41
- dtype=init_dtype)
42
- w2 = torch.randn(num_experts,
43
- hidden_size,
44
- shard_intermediate_size // 2,
45
- dtype=init_dtype)
39
+ if use_int8_w8a16:
40
+ w1 = torch.randint(-127,
41
+ 127, (
42
+ num_experts,
43
+ shard_intermediate_size,
44
+ hidden_size,
45
+ ),
46
+ dtype=torch.int8)
47
+ w2 = torch.randint(-127,
48
+ 127, (
49
+ num_experts,
50
+ hidden_size,
51
+ shard_intermediate_size // 2,
52
+ ),
53
+ dtype=torch.int8)
54
+ else:
55
+ w1 = torch.randn(num_experts,
56
+ shard_intermediate_size,
57
+ hidden_size,
58
+ dtype=init_dtype)
59
+ w2 = torch.randn(num_experts,
60
+ hidden_size,
61
+ shard_intermediate_size // 2,
62
+ dtype=init_dtype)
63
gating_output = torch.randn(num_iters,
64
num_tokens,
65
num_experts,
@@ -52,7 +69,11 @@ def benchmark_config(
69
w2_scale = None
70
a1_scale = None
71
a2_scale = None
55
- if use_fp8:
72
+ if use_int8_w8a16:
73
+ w1_scale = torch.randn((num_experts, 2 * shard_intermediate_size),
74
+ dtype=torch.float32)
75
+ w2_scale = torch.randn((hidden_size, num_experts), dtype=torch.float32)
76
+ if use_fp8_w8a8:
77
w1_scale = torch.randn(num_experts, dtype=torch.float32)
78
w2_scale = torch.randn(num_experts, dtype=torch.float32)
79
a1_scale = torch.randn(1, dtype=torch.float32)
@@ -76,7 +97,8 @@ def benchmark_config(
97
renormalize=True,
98
inplace=True,
99
override_config=config,
79
- use_fp8=use_fp8,
100
+ use_fp8_w8a8=use_fp8_w8a8,
101
+ use_int8_w8a16=use_int8_w8a16,
102
w1_scale=w1_scale,
103
w2_scale=w2_scale,
104
a1_scale=a1_scale,
@@ -155,11 +177,13 @@ class BenchmarkWorker:
177
hidden_size: int,
178
topk: int,
179
dtype: torch.dtype,
158
- use_fp8: bool,
180
+ use_fp8_w8a8: bool,
181
+ use_int8_w8a16: bool,
182
) -> Tuple[Dict[str, int], float]:
183
torch.cuda.manual_seed_all(self.seed)
161
-
162
- dtype_str = "float8" if use_fp8 else None
184
+ dtype_str = get_config_dtype_str(dtype,
185
+ use_int8_w8a16=use_int8_w8a16,
186
+ use_fp8_w8a8=use_fp8_w8a8)
187
# NOTE(woosuk): The current naming convention uses w2.shape[2], which
188
# is the intermediate size after silu_and_mul.
189
op_config = get_moe_configs(num_experts, shard_intermediate_size // 2,
@@ -173,7 +197,8 @@ class BenchmarkWorker:
197
key=lambda x: abs(x - num_tokens))]
198
kernel_time = benchmark_config(config, num_tokens, num_experts,
199
shard_intermediate_size, hidden_size,
176
- topk, dtype, use_fp8)
200
+ topk, dtype, use_fp8_w8a8,
201
+ use_int8_w8a16)
202
return config, kernel_time
203
204
def tune(
@@ -184,9 +209,10 @@ class BenchmarkWorker:
209
hidden_size: int,
210
topk: int,
211
dtype: torch.dtype,
187
- use_fp8: bool,
188
- search_space: List[BenchmarkConfig],
189
- ) -> BenchmarkConfig:
212
+ use_fp8_w8a8: bool,
213
+ use_int8_w8a16: bool,
214
+ search_space: List[Dict[str, int]],
215
+ ) -> Dict[str, int]:
216
best_config = None
217
best_time = float("inf")
218
for config in tqdm(search_space):
@@ -198,7 +224,8 @@ class BenchmarkWorker:
224
hidden_size,
225
topk,
226
dtype,
201
- use_fp8,
227
+ use_fp8_w8a8,
228
+ use_int8_w8a16,
229
num_iters=10)
230
except triton.runtime.autotuner.OutOfResources:
231
# Some configurations may be invalid and fail to compile.
@@ -224,20 +251,19 @@ def sort_config(config: BenchmarkConfig) -> BenchmarkConfig:
251
}
252
253
227
-def save_configs(
228
- configs: Dict[int, BenchmarkConfig],
229
- num_experts: int,
230
- shard_intermediate_size: int,
231
- hidden_size: int,
232
- topk: int,
233
- dtype: torch.dtype,
234
- use_fp8: bool,
235
-) -> None:
236
- dtype_str = "float8" if use_fp8 else None
254
+def save_configs(configs: Dict[int, BenchmarkConfig], num_experts: int,
255
+ shard_intermediate_size: int, hidden_size: int, topk: int,
256
+ dtype: torch.dtype, use_fp8_w8a8: bool,
257
+ use_int8_w8a16: bool) -> None:
258
+ dtype_str = get_config_dtype_str(dtype,
259
+ use_int8_w8a16=use_int8_w8a16,
260
+ use_fp8_w8a8=use_fp8_w8a8)
261
+
262
# NOTE(woosuk): The current naming convention uses w2.shape[2], which
263
# is the intermediate size after silu_and_mul.
264
filename = get_config_file_name(num_experts, shard_intermediate_size // 2,
265
dtype_str)
266
+
267
print(f"Writing best config to {filename}...")
268
with open(filename, "w") as f:
269
json.dump(configs, f, indent=4)
@@ -253,6 +279,11 @@ def main(args: argparse.Namespace):
279
topk = config.ffn_config.moe_top_k
280
intermediate_size = config.ffn_config.ffn_hidden_size
281
shard_intermediate_size = 2 * intermediate_size // args.tp_size
282
+ elif config.architectures[0] == "JambaForCausalLM":
283
+ E = config.num_experts
284
+ topk = config.num_experts_per_tok
285
+ intermediate_size = config.intermediate_size
286
+ shard_intermediate_size = 2 * intermediate_size // args.tp_size
287
else:
288
# Default: Mixtral.
289
E = config.num_local_experts
@@ -262,7 +293,8 @@ def main(args: argparse.Namespace):
293
294
hidden_size = config.hidden_size
295
dtype = config.torch_dtype
265
- use_fp8 = args.dtype == "fp8"
296
+ use_fp8_w8a8 = args.dtype == "fp8_w8a8"
297
+ use_int8_w8a16 = args.dtype == "int8_w8a16"
298
299
if args.batch_size is None:
300
batch_sizes = [
@@ -294,21 +326,21 @@ def main(args: argparse.Namespace):
326
start = time.time()
327
configs = _distribute(
328
"tune", [(batch_size, E, shard_intermediate_size, hidden_size,
297
- topk, dtype, use_fp8, search_space)
329
+ topk, dtype, use_fp8_w8a8, use_int8_w8a16, search_space)
330
for batch_size in batch_sizes])
331
best_configs = {
332
M: sort_config(config)
333
for M, config in zip(batch_sizes, configs)
334
}
335
save_configs(best_configs, E, shard_intermediate_size, hidden_size,
304
- topk, dtype, use_fp8)
336
+ topk, dtype, use_fp8_w8a8, use_int8_w8a16)
337
end = time.time()
338
print(f"Tuning took {end - start:.2f} seconds")
339
else:
308
- outputs = _distribute("benchmark",
309
- [(batch_size, E, shard_intermediate_size,
310
- hidden_size, topk, dtype, use_fp8)
311
- for batch_size in batch_sizes])
340
+ outputs = _distribute(
341
+ "benchmark", [(batch_size, E, shard_intermediate_size, hidden_size,
342
+ topk, dtype, use_fp8_w8a8, use_int8_w8a16)
343
+ for batch_size in batch_sizes])
344
345
for batch_size, (config, kernel_time) in zip(batch_sizes, outputs):
346
print(f"Batch size: {batch_size}, config: {config}")
@@ -323,7 +355,7 @@ if __name__ == "__main__":
355
parser.add_argument("--tp-size", "-tp", type=int, default=2)
356
parser.add_argument("--dtype",
357
type=str,
326
- choices=["auto", "fp8"],
358
+ choices=["auto", "fp8_w8a8", "int8_w8a16"],
359
default="auto")
360
parser.add_argument("--seed", type=int, default=0)
361
parser.add_argument("--batch-size", type=int, required=False)
tests/models/test_jamba.py
+7
-6
@@ -6,9 +6,12 @@ from vllm.worker.model_runner import _get_graph_batch_size
6
MODELS = ["ai21labs/Jamba-tiny-random"]
7
8
9
+# Fails due to usage of MoE as MLP(E=1_, which is different than the HF impl
10
+# TODO: Fix this with trained model
11
+@pytest.mark.skip()
12
@pytest.mark.parametrize("model", MODELS)
10
-@pytest.mark.parametrize("dtype", ["float"])
11
-@pytest.mark.parametrize("max_tokens", [20])
13
+@pytest.mark.parametrize("dtype", ["bfloat16"])
14
+@pytest.mark.parametrize("max_tokens", [10])
15
def test_models(
16
hf_runner,
17
vllm_runner,
@@ -17,8 +20,6 @@ def test_models(
20
dtype: str,
21
max_tokens: int,
22
) -> None:
20
- # To pass the small model tests, we need full precision.
21
- assert dtype == "float"
23
24
with hf_runner(model, dtype=dtype) as hf_model:
25
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
@@ -36,8 +37,8 @@ def test_models(
37
38
39
@pytest.mark.parametrize("model", MODELS)
39
-@pytest.mark.parametrize("dtype", ["float"])
40
-@pytest.mark.parametrize("max_tokens", [20])
40
+@pytest.mark.parametrize("dtype", ["half"])
41
+@pytest.mark.parametrize("max_tokens", [5])
42
def test_batching(
43
vllm_runner,
44
example_prompts,
tests/quantization/test_experts_int8.py
new
+28
@@ -0,0 +1,28 @@
1
+# flake8: noqa
2
+"""Tests experts_int8 quantization startup and generation,
3
+doesn't test correctness
4
+"""
5
+import pytest
6
+
7
+from tests.quantization.utils import is_quant_method_supported
8
+
9
+MODELS = ["ai21labs/Jamba-tiny-random"]
10
+
11
+
12
+@pytest.mark.skipif(not is_quant_method_supported("experts_int8"),
13
+ reason="ExpertsInt8 is not supported on this GPU type.")
14
+@pytest.mark.parametrize("model", MODELS)
15
+@pytest.mark.parametrize("dtype", ["bfloat16"])
16
+@pytest.mark.parametrize("max_tokens", [10])
17
+def test_model_experts_int8_startup(
18
+ hf_runner,
19
+ vllm_runner,
20
+ example_prompts,
21
+ model: str,
22
+ dtype: str,
23
+ max_tokens: int,
24
+) -> None:
25
+
26
+ with vllm_runner(model, dtype=dtype,
27
+ quantization="experts_int8") as vllm_model:
28
+ vllm_model.generate_greedy(example_prompts, max_tokens)
vllm/config.py
+2
-1
@@ -243,7 +243,8 @@ class ModelConfig:
243
rocm_supported_quantization = ["gptq", "squeezellm", "fp8"]
244
optimized_quantization_methods = [
245
"fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
246
- "fbgemm_fp8", "compressed_tensors", "compressed-tensors"
246
+ "fbgemm_fp8", "compressed_tensors", "compressed-tensors",
247
+ "experts_int8"
248
]
249
tpu_supported_quantization = ["tpu_int8"]
250
if self.quantization is not None:
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json
renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json
renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json
renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json
renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json
renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json
renamed
vllm/model_executor/layers/fused_moe/fused_moe.py
+91
-53
@@ -17,42 +17,44 @@ logger = init_logger(__name__)
17
18
@triton.jit
19
def fused_moe_kernel(
20
- # Pointers to matrices
21
- a_ptr,
22
- b_ptr,
23
- c_ptr,
24
- a_scale_ptr,
25
- b_scale_ptr,
26
- topk_weights_ptr,
27
- sorted_token_ids_ptr,
28
- expert_ids_ptr,
29
- num_tokens_post_padded_ptr,
30
- # Matrix dimensions
31
- N,
32
- K,
33
- EM,
34
- num_valid_tokens,
35
- # The stride variables represent how much to increase the ptr by when
36
- # moving by 1 element in a particular dimension. E.g. `stride_am` is
37
- # how much to increase `a_ptr` by to get the element one row down
38
- # (A has M rows).
39
- stride_am,
40
- stride_ak,
41
- stride_be,
42
- stride_bk,
43
- stride_bn,
44
- stride_cm,
45
- stride_cn,
46
- # Meta-parameters
47
- BLOCK_SIZE_M: tl.constexpr,
48
- BLOCK_SIZE_N: tl.constexpr,
49
- BLOCK_SIZE_K: tl.constexpr,
50
- GROUP_SIZE_M: tl.constexpr,
51
- MUL_ROUTED_WEIGHT: tl.constexpr,
52
- top_k: tl.constexpr,
53
- compute_type: tl.constexpr,
54
- use_fp8: tl.constexpr,
55
-):
20
+ # Pointers to matrices
21
+ a_ptr,
22
+ b_ptr,
23
+ c_ptr,
24
+ a_scale_ptr,
25
+ b_scale_ptr,
26
+ topk_weights_ptr,
27
+ sorted_token_ids_ptr,
28
+ expert_ids_ptr,
29
+ num_tokens_post_padded_ptr,
30
+ # Matrix dimensions
31
+ N,
32
+ K,
33
+ EM,
34
+ num_valid_tokens,
35
+ # The stride variables represent how much to increase the ptr by when
36
+ # moving by 1 element in a particular dimension. E.g. `stride_am` is
37
+ # how much to increase `a_ptr` by to get the element one row down
38
+ # (A has M rows).
39
+ stride_am,
40
+ stride_ak,
41
+ stride_be,
42
+ stride_bk,
43
+ stride_bn,
44
+ stride_cm,
45
+ stride_cn,
46
+ stride_bse,
47
+ stride_bsn,
48
+ # Meta-parameters
49
+ BLOCK_SIZE_M: tl.constexpr,
50
+ BLOCK_SIZE_N: tl.constexpr,
51
+ BLOCK_SIZE_K: tl.constexpr,
52
+ GROUP_SIZE_M: tl.constexpr,
53
+ MUL_ROUTED_WEIGHT: tl.constexpr,
54
+ top_k: tl.constexpr,
55
+ compute_type: tl.constexpr,
56
+ use_fp8_w8a8: tl.constexpr,
57
+ use_int8_w8a16: tl.constexpr):
58
"""
59
Implements the fused computation for a Mixture of Experts (MOE) using
60
token and expert matrices.
@@ -113,8 +115,12 @@ def fused_moe_kernel(
115
off_experts = tl.load(expert_ids_ptr + pid_m)
116
b_ptrs = b_ptr + off_experts * stride_be + (offs_k[:, None] * stride_bk +
117
offs_bn[None, :] * stride_bn)
118
+ if use_int8_w8a16:
119
+ b_scale_ptrs = b_scale_ptr + off_experts * stride_bse + offs_bn[
120
+ None, :] * stride_bsn
121
+ b_scale = tl.load(b_scale_ptrs)
122
117
- if use_fp8:
123
+ if use_fp8_w8a8:
124
a_scale = tl.load(a_scale_ptr)
125
b_scale = tl.load(b_scale_ptr + off_experts)
126
@@ -136,7 +142,9 @@ def fused_moe_kernel(
142
mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
143
other=0.0)
144
# We accumulate along the K dimension.
139
- if use_fp8:
145
+ if use_int8_w8a16:
146
+ accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
147
+ elif use_fp8_w8a8:
148
accumulator = tl.dot(a, b, acc=accumulator)
149
else:
150
accumulator += tl.dot(a, b)
@@ -149,8 +157,9 @@ def fused_moe_kernel(
157
mask=token_mask,
158
other=0)
159
accumulator = accumulator * moe_weight[:, None]
152
-
153
- if use_fp8:
160
+ if use_int8_w8a16:
161
+ accumulator = (accumulator * b_scale).to(compute_type)
162
+ elif use_fp8_w8a8:
163
accumulator = (accumulator * a_scale * b_scale).to(compute_type)
164
else:
165
accumulator = accumulator.to(compute_type)
@@ -229,16 +238,18 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
238
num_tokens_post_padded: torch.Tensor,
239
mul_routed_weight: bool, top_k: int,
240
config: Dict[str, Any], compute_type: tl.dtype,
232
- use_fp8: bool) -> None:
241
+ use_fp8_w8a8: bool, use_int8_w8a16: bool) -> None:
242
assert topk_weights.stride(1) == 1
243
assert sorted_token_ids.stride(0) == 1
244
236
- if not use_fp8:
237
- assert A_scale is None
238
- assert B_scale is None
239
- else:
245
+ if use_fp8_w8a8:
246
A, A_scale = ops.scaled_fp8_quant(A, A_scale)
247
assert B_scale is not None
248
+ elif use_int8_w8a16:
249
+ assert B_scale is not None
250
+ else:
251
+ assert A_scale is None
252
+ assert B_scale is None
253
254
grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[
255
'BLOCK_SIZE_M']) * triton.cdiv(B.shape[1], META['BLOCK_SIZE_N']), )
@@ -264,10 +275,13 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
275
B.stride(1),
276
C.stride(1),
277
C.stride(2),
278
+ B_scale.stride(0) if B_scale is not None and use_int8_w8a16 else 0,
279
+ B_scale.stride(1) if B_scale is not None and use_int8_w8a16 else 0,
280
MUL_ROUTED_WEIGHT=mul_routed_weight,
281
top_k=top_k,
282
compute_type=compute_type,
270
- use_fp8=use_fp8,
283
+ use_fp8_w8a8=use_fp8_w8a8,
284
+ use_int8_w8a16=use_int8_w8a16,
285
**config,
286
)
287
@@ -426,6 +440,20 @@ def grouped_topk(hidden_states: torch.Tensor,
440
return topk_weights, topk_ids
441
442
443
+def get_config_dtype_str(dtype: torch.dtype,
444
+ use_int8_w8a16: Optional[bool] = False,
445
+ use_fp8_w8a8: Optional[bool] = False):
446
+ if use_fp8_w8a8:
447
+ return "fp8_w8a8"
448
+ elif use_int8_w8a16:
449
+ return "int8_w8a16"
450
+ elif dtype == torch.float:
451
+ # avoiding cases where kernel fails when float32 MoE
452
+ # use fp16/bfloat16 configs
453
+ return "float32"
454
+ return None
455
+
456
+
457
def fused_experts(hidden_states: torch.Tensor,
458
w1: torch.Tensor,
459
w2: torch.Tensor,
@@ -433,7 +461,8 @@ def fused_experts(hidden_states: torch.Tensor,
461
topk_ids: torch.Tensor,
462
inplace: bool = False,
463
override_config: Optional[Dict[str, Any]] = None,
436
- use_fp8: bool = False,
464
+ use_fp8_w8a8: bool = False,
465
+ use_int8_w8a16: bool = False,
466
w1_scale: Optional[torch.Tensor] = None,
467
w2_scale: Optional[torch.Tensor] = None,
468
a1_scale: Optional[torch.Tensor] = None,
@@ -454,13 +483,16 @@ def fused_experts(hidden_states: torch.Tensor,
483
# https://github.com/vllm-project/vllm/issues/5938
484
CHUNK_SIZE = envs.VLLM_FUSED_MOE_CHUNK_SIZE
485
M = min(num_tokens, CHUNK_SIZE)
486
+ config_dtype = get_config_dtype_str(use_fp8_w8a8=use_fp8_w8a8,
487
+ use_int8_w8a16=use_int8_w8a16,
488
+ dtype=hidden_states.dtype)
489
490
get_config_func = functools.partial(
491
try_get_optimal_moe_config,
492
w1.shape,
493
w2.shape,
494
topk_ids.shape[1],
463
- "float8" if use_fp8 else None,
495
+ config_dtype,
496
override_config=override_config,
497
)
498
@@ -524,7 +556,8 @@ def fused_experts(hidden_states: torch.Tensor,
556
topk_ids.shape[1],
557
config,
558
compute_type=compute_type,
527
- use_fp8=use_fp8)
559
+ use_fp8_w8a8=use_fp8_w8a8,
560
+ use_int8_w8a16=use_int8_w8a16)
561
562
ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
563
@@ -542,7 +575,8 @@ def fused_experts(hidden_states: torch.Tensor,
575
1,
576
config,
577
compute_type=compute_type,
545
- use_fp8=use_fp8)
578
+ use_fp8_w8a8=use_fp8_w8a8,
579
+ use_int8_w8a16=use_int8_w8a16)
580
581
torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
582
dim=1,
@@ -562,7 +596,8 @@ def fused_moe(
596
use_grouped_topk: bool = False,
597
num_expert_group: Optional[int] = None,
598
topk_group: Optional[int] = None,
565
- use_fp8: bool = False,
599
+ use_fp8_w8a8: bool = False,
600
+ use_int8_w8a16: bool = False,
601
w1_scale: Optional[torch.Tensor] = None,
602
w2_scale: Optional[torch.Tensor] = None,
603
a1_scale: Optional[torch.Tensor] = None,
@@ -588,7 +623,9 @@ def fused_moe(
623
- topk_group: Optional[int]: additional parameter for grouped_topk
624
- use_grouped_topk: If True, use grouped_topk instead of fused_topk
625
note: Deepseekv2 model uses grouped_topk
591
- - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
626
+ - use_fp8_w8a8 (bool): If True, use fp8 arithmetic to compute the inner
627
+ products for w1 and w2. Defaults to False.
628
+ - use_int8_w8a16 (bool): If True, use fp8 arithmetic to compute the inner
629
products for w1 and w2. Defaults to False.
630
- w1_scale (Optional[torch.Tensor]): Optional scale to be used for
631
w1.
@@ -617,7 +654,8 @@ def fused_moe(
654
topk_ids,
655
inplace=inplace,
656
override_config=override_config,
620
- use_fp8=use_fp8,
657
+ use_fp8_w8a8=use_fp8_w8a8,
658
+ use_int8_w8a16=use_int8_w8a16,
659
w1_scale=w1_scale,
660
w2_scale=w2_scale,
661
a1_scale=a1_scale,
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -11,6 +11,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
11
CompressedTensorsConfig)
12
from vllm.model_executor.layers.quantization.deepspeedfp import (
13
DeepSpeedFPConfig)
14
+from vllm.model_executor.layers.quantization.experts_int8 import (
15
+ ExpertsInt8Config)
16
from vllm.model_executor.layers.quantization.fbgemm_fp8 import FBGEMMFp8Config
17
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
18
from vllm.model_executor.layers.quantization.gguf import GGUFConfig
@@ -43,6 +45,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
45
"compressed-tensors": CompressedTensorsConfig,
46
"bitsandbytes": BitsAndBytesConfig,
47
"qqq": QQQConfig,
48
+ "experts_int8": ExpertsInt8Config,
49
}
50
51
vllm/model_executor/layers/quantization/experts_int8.py
new
+175
@@ -0,0 +1,175 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+
5
+from vllm.distributed import get_tensor_model_parallel_rank, get_tp_group
6
+from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
7
+from vllm.model_executor.layers.linear import (LinearBase,
8
+ UnquantizedLinearMethod)
9
+from vllm.model_executor.layers.quantization.base_config import (
10
+ QuantizationConfig, QuantizeMethodBase)
11
+from vllm.model_executor.utils import set_weight_attrs
12
+
13
+
14
+class ExpertsInt8Config(QuantizationConfig):
15
+ """Config class for Int8 experts quantization."""
16
+
17
+ def __init__(self) -> None:
18
+ pass
19
+
20
+ @classmethod
21
+ def get_name(cls) -> str:
22
+ return "experts_int8"
23
+
24
+ @classmethod
25
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
26
+ return [torch.bfloat16, torch.half]
27
+
28
+ @classmethod
29
+ def get_min_capability(cls) -> int:
30
+ return 80
31
+
32
+ @classmethod
33
+ def get_config_filenames(cls) -> List[str]:
34
+ return []
35
+
36
+ @classmethod
37
+ def from_config(cls, config: Dict[str, Any]) -> "ExpertsInt8Config":
38
+ return cls()
39
+
40
+ def get_quant_method(self, layer: torch.nn.Module,
41
+ prefix: str) -> Optional["QuantizeMethodBase"]:
42
+ if isinstance(layer, LinearBase):
43
+ return UnquantizedLinearMethod()
44
+ elif isinstance(layer, FusedMoE):
45
+ return ExpertsInt8MoEMethod(self)
46
+ return None
47
+
48
+ def get_scaled_act_names(self) -> List[str]:
49
+ return []
50
+
51
+
52
+class ExpertsInt8MoEMethod(FusedMoEMethodBase):
53
+
54
+ def __init__(self, quant_config: ExpertsInt8Config):
55
+ self.quant_config = quant_config
56
+
57
+ def create_weights(self, layer: torch.nn.Module, num_experts: int,
58
+ hidden_size: int, intermediate_size: int,
59
+ params_dtype: torch.dtype, **extra_weight_attrs):
60
+
61
+ int8_dtype = torch.int8
62
+
63
+ assert 'weight_loader' in extra_weight_attrs
64
+ weight_loader = extra_weight_attrs['weight_loader']
65
+ wrapped_weight_loader = ExpertsInt8MoEMethod.quantizing_weight_loader(
66
+ layer, weight_loader)
67
+ extra_weight_attrs['weight_loader'] = wrapped_weight_loader
68
+
69
+ # Fused gate_up_proj (column parallel)
70
+ w13_weight = torch.nn.Parameter(torch.empty(num_experts,
71
+ 2 * intermediate_size,
72
+ hidden_size,
73
+ dtype=int8_dtype),
74
+ requires_grad=False)
75
+ layer.register_parameter("w13_weight", w13_weight)
76
+ set_weight_attrs(w13_weight, extra_weight_attrs)
77
+
78
+ # down_proj (row parallel)
79
+ w2_weight = torch.nn.Parameter(torch.empty(num_experts,
80
+ hidden_size,
81
+ intermediate_size,
82
+ dtype=int8_dtype),
83
+ requires_grad=False)
84
+ layer.register_parameter("w2_weight", w2_weight)
85
+ set_weight_attrs(w2_weight, extra_weight_attrs)
86
+
87
+ w13_scale = torch.nn.Parameter(torch.zeros(num_experts,
88
+ 2 * intermediate_size,
89
+ dtype=torch.float32),
90
+ requires_grad=False)
91
+ layer.register_parameter("w13_scale", w13_scale)
92
+
93
+ w2_scale = torch.nn.Parameter(torch.zeros(num_experts,
94
+ hidden_size,
95
+ dtype=torch.float32),
96
+ requires_grad=False)
97
+ layer.register_parameter("w2_scale", w2_scale)
98
+
99
+ def apply(self,
100
+ layer: torch.nn.Module,
101
+ x: torch.Tensor,
102
+ router_logits: torch.Tensor,
103
+ top_k: int,
104
+ renormalize: bool = True,
105
+ use_grouped_topk: bool = False,
106
+ num_expert_group: Optional[int] = None,
107
+ topk_group: Optional[int] = None) -> torch.Tensor:
108
+ from vllm.model_executor.layers.fused_moe import fused_experts
109
+
110
+ topk_weights, topk_ids = FusedMoE.select_experts(
111
+ hidden_states=x,
112
+ router_logits=router_logits,
113
+ use_grouped_topk=use_grouped_topk,
114
+ top_k=top_k,
115
+ renormalize=renormalize,
116
+ topk_group=topk_group,
117
+ num_expert_group=num_expert_group)
118
+
119
+ return fused_experts(x,
120
+ layer.w13_weight,
121
+ layer.w2_weight,
122
+ topk_weights=topk_weights,
123
+ topk_ids=topk_ids,
124
+ inplace=True,
125
+ use_int8_w8a16=True,
126
+ w1_scale=layer.w13_scale,
127
+ w2_scale=layer.w2_scale)
128
+
129
+ @staticmethod
130
+ def quantizing_weight_loader(layer, weight_loader):
131
+
132
+ def quantize_and_call_weight_loader(param: torch.nn.Parameter,
133
+ loaded_weight: torch.Tensor,
134
+ weight_name: str, shard_id: int,
135
+ expert_id: int):
136
+ tp_rank = get_tensor_model_parallel_rank()
137
+ shard_size = layer.intermediate_size_per_partition
138
+ shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
139
+ device = get_tp_group().device
140
+ loaded_weight = loaded_weight.to(device)
141
+ # w1, gate_proj case: Load into first shard of w13.
142
+ if shard_id == "w1":
143
+ scales = quantize_in_place_and_get_scales(
144
+ loaded_weight[shard, :])
145
+ layer.w13_scale.data[expert_id, 0:shard_size].copy_(scales[:,
146
+ 0])
147
+ # w3, up_proj case: Load into second shard of w13.
148
+ elif shard_id == "w3":
149
+ scales = quantize_in_place_and_get_scales(
150
+ loaded_weight[shard, :])
151
+ layer.w13_scale.data[expert_id, shard_size:2 *
152
+ shard_size].copy_(scales[:, 0])
153
+ # w2, down_proj case: Load into only shard of w2.
154
+ elif shard_id == "w2":
155
+ scales = quantize_in_place_and_get_scales(loaded_weight[:,
156
+ shard])
157
+ layer.w2_scale.data[expert_id, :].copy_(scales[:, 0])
158
+ else:
159
+ raise ValueError(
160
+ f"Shard id must be in [0,1,2] but got {shard_id}")
161
+ weight_loader(param, loaded_weight, weight_name, shard_id,
162
+ expert_id)
163
+
164
+ return quantize_and_call_weight_loader
165
+
166
+
167
+def quantize_in_place_and_get_scales(weight: torch.Tensor) -> torch.Tensor:
168
+ vmax = torch.iinfo(torch.int8).max
169
+ scales = (torch.max(torch.abs(weight), dim=1, keepdim=True)[0] / vmax)
170
+
171
+ weight.div_(scales)
172
+ weight.round_()
173
+ weight.clamp_(-vmax, vmax)
174
+
175
+ return scales
vllm/model_executor/layers/quantization/fp8.py
+1
-1
@@ -488,7 +488,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
488
topk_weights=topk_weights,
489
topk_ids=topk_ids,
490
inplace=True,
491
- use_fp8=True,
491
+ use_fp8_w8a8=True,
492
w1_scale=layer.w13_weight_scale,
493
w2_scale=layer.w2_weight_scale,
494
a1_scale=layer.w13_input_scale,
vllm/model_executor/models/jamba.py
+35
-37
@@ -16,7 +16,6 @@ from vllm.attention.layer import Attention
16
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
17
from vllm.distributed import (get_tensor_model_parallel_rank,
18
get_tensor_model_parallel_world_size)
19
-from vllm.model_executor.layers.activation import SiluAndMul
19
from vllm.model_executor.layers.fused_moe import FusedMoE
20
from vllm.model_executor.layers.layernorm import RMSNorm
21
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -249,37 +248,6 @@ class JambaMambaMixer(nn.Module):
248
return hidden_states
249
250
252
-class JambaMLP(nn.Module):
253
-
254
- def __init__(
255
- self,
256
- config: JambaConfig,
257
- quant_config: Optional[QuantizationConfig] = None,
258
- ) -> None:
259
- super().__init__()
260
- hidden_size = config.hidden_size
261
- intermediate_size = config.intermediate_size
262
- hidden_act = config.hidden_act
263
- self.gate_up_proj = MergedColumnParallelLinear(
264
- hidden_size, [intermediate_size] * 2,
265
- bias=False,
266
- quant_config=quant_config)
267
- self.down_proj = RowParallelLinear(intermediate_size,
268
- hidden_size,
269
- bias=False,
270
- quant_config=quant_config)
271
- if hidden_act != "silu":
272
- raise ValueError(f"Unsupported activation: {hidden_act}. "
273
- "Only silu is supported for now.")
274
- self.act_fn = SiluAndMul()
275
-
276
- def forward(self, x):
277
- gate_up, _ = self.gate_up_proj(x)
278
- x = self.act_fn(gate_up)
279
- x, _ = self.down_proj(x)
280
- return x
281
-
282
-
251
class JambaMoE(nn.Module):
252
253
def __init__(self,
@@ -327,6 +295,21 @@ class JambaMoE(nn.Module):
295
return hidden_states.view(orig_shape)
296
297
298
+class JambaMLP(JambaMoE):
299
+
300
+ def __init__(self,
301
+ config: JambaConfig,
302
+ params_dtype: Optional[torch.dtype] = None,
303
+ tp_size: Optional[int] = None,
304
+ quant_config: Optional[QuantizationConfig] = None):
305
+ super().__init__(config,
306
+ num_experts=1,
307
+ top_k=1,
308
+ params_dtype=params_dtype,
309
+ tp_size=tp_size,
310
+ quant_config=quant_config)
311
+
312
+
313
class JambaMambaDecoderLayer(nn.Module):
314
315
def __init__(self,
@@ -884,8 +867,6 @@ class JambaForCausalLM(nn.Module, HasInnerState):
867
("qkv_proj", "q_proj", "q"),
868
("qkv_proj", "k_proj", "k"),
869
("qkv_proj", "v_proj", "v"),
887
- ("gate_up_proj", "gate_proj", 0),
888
- ("gate_up_proj", "up_proj", 1),
870
]
871
872
# Params for weights, fp8 weight scales, fp8 activation scales
@@ -907,6 +888,10 @@ class JambaForCausalLM(nn.Module, HasInnerState):
888
if ".self_attn." in name:
889
name = name.replace(".self_attn", "")
890
891
+ if "feed_forward" in name and not _is_moe_layer(name):
892
+ ## map MLP layers to expert with ID=0
893
+ name = name.replace("feed_forward", "feed_forward.experts.0")
894
+
895
for param_name, weight_name, shard_id in stacked_params_mapping:
896
if weight_name not in name:
897
continue
@@ -921,16 +906,21 @@ class JambaForCausalLM(nn.Module, HasInnerState):
906
weight_loader(param, loaded_weight, shard_id)
907
break
908
else:
924
- for mapping in expert_params_mapping:
925
- param_name, weight_name, expert_id, shard_id = mapping
909
+ for (
910
+ param_name,
911
+ weight_name,
912
+ expert_id,
913
+ shard_id,
914
+ ) in expert_params_mapping:
915
if weight_name not in name:
916
continue
917
+
918
name = name.replace(weight_name, param_name)
919
param = params_dict[name]
920
weight_loader = param.weight_loader
921
weight_loader(param,
922
loaded_weight,
933
- name,
923
+ weight_name,
924
shard_id=shard_id,
925
expert_id=expert_id)
926
break
@@ -943,3 +933,11 @@ class JambaForCausalLM(nn.Module, HasInnerState):
933
weight_loader = getattr(param, "weight_loader",
934
default_weight_loader)
935
weight_loader(param, loaded_weight)
936
+
937
+
938
+def _is_moe_layer(name: str):
939
+ return any(
940
+ [experts_name in name for experts_name in [
941
+ "experts",
942
+ "router",
943
+ ]])