@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7fc23be81c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Kernel] W8A16 Int8 inside FusedMoE (#7415)

Mor Zusman committed Aug 16, 2024 at 20:06 UTC 7fc23be81c55ca0570f551871a3adc994aaefc05
15 files changed +412 -136
benchmarks/kernels/benchmark_moe.py
+70 -38
@@ -30,19 +30,36 @@ def benchmark_config(
30 hidden_size: int,
31 topk: int,
32 dtype: torch.dtype,
33 - use_fp8: bool,
33 + use_fp8_w8a8: bool,
34 + use_int8_w8a16: bool,
35 num_iters: int = 100,
36 ) -> float:
36 - init_dtype = torch.float16 if use_fp8 else dtype
37 + init_dtype = torch.float16 if use_fp8_w8a8 else dtype
38 x = torch.randn(num_tokens, hidden_size, dtype=dtype)
38 - w1 = torch.randn(num_experts,
39 - shard_intermediate_size,
40 - hidden_size,
41 - dtype=init_dtype)
42 - w2 = torch.randn(num_experts,
43 - hidden_size,
44 - shard_intermediate_size // 2,
45 - dtype=init_dtype)
39 + if use_int8_w8a16:
40 + w1 = torch.randint(-127,
41 + 127, (
42 + num_experts,
43 + shard_intermediate_size,
44 + hidden_size,
45 + ),
46 + dtype=torch.int8)
47 + w2 = torch.randint(-127,
48 + 127, (
49 + num_experts,
50 + hidden_size,
51 + shard_intermediate_size // 2,
52 + ),
53 + dtype=torch.int8)
54 + else:
55 + w1 = torch.randn(num_experts,
56 + shard_intermediate_size,
57 + hidden_size,
58 + dtype=init_dtype)
59 + w2 = torch.randn(num_experts,
60 + hidden_size,
61 + shard_intermediate_size // 2,
62 + dtype=init_dtype)
63 gating_output = torch.randn(num_iters,
64 num_tokens,
65 num_experts,
@@ -52,7 +69,11 @@ def benchmark_config(
69 w2_scale = None
70 a1_scale = None
71 a2_scale = None
55 - if use_fp8:
72 + if use_int8_w8a16:
73 + w1_scale = torch.randn((num_experts, 2 * shard_intermediate_size),
74 + dtype=torch.float32)
75 + w2_scale = torch.randn((hidden_size, num_experts), dtype=torch.float32)
76 + if use_fp8_w8a8:
77 w1_scale = torch.randn(num_experts, dtype=torch.float32)
78 w2_scale = torch.randn(num_experts, dtype=torch.float32)
79 a1_scale = torch.randn(1, dtype=torch.float32)
@@ -76,7 +97,8 @@ def benchmark_config(
97 renormalize=True,
98 inplace=True,
99 override_config=config,
79 - use_fp8=use_fp8,
100 + use_fp8_w8a8=use_fp8_w8a8,
101 + use_int8_w8a16=use_int8_w8a16,
102 w1_scale=w1_scale,
103 w2_scale=w2_scale,
104 a1_scale=a1_scale,
@@ -155,11 +177,13 @@ class BenchmarkWorker:
177 hidden_size: int,
178 topk: int,
179 dtype: torch.dtype,
158 - use_fp8: bool,
180 + use_fp8_w8a8: bool,
181 + use_int8_w8a16: bool,
182 ) -> Tuple[Dict[str, int], float]:
183 torch.cuda.manual_seed_all(self.seed)
161 -
162 - dtype_str = "float8" if use_fp8 else None
184 + dtype_str = get_config_dtype_str(dtype,
185 + use_int8_w8a16=use_int8_w8a16,
186 + use_fp8_w8a8=use_fp8_w8a8)
187 # NOTE(woosuk): The current naming convention uses w2.shape[2], which
188 # is the intermediate size after silu_and_mul.
189 op_config = get_moe_configs(num_experts, shard_intermediate_size // 2,
@@ -173,7 +197,8 @@ class BenchmarkWorker:
197 key=lambda x: abs(x - num_tokens))]
198 kernel_time = benchmark_config(config, num_tokens, num_experts,
199 shard_intermediate_size, hidden_size,
176 - topk, dtype, use_fp8)
200 + topk, dtype, use_fp8_w8a8,
201 + use_int8_w8a16)
202 return config, kernel_time
203
204 def tune(
@@ -184,9 +209,10 @@ class BenchmarkWorker:
209 hidden_size: int,
210 topk: int,
211 dtype: torch.dtype,
187 - use_fp8: bool,
188 - search_space: List[BenchmarkConfig],
189 - ) -> BenchmarkConfig:
212 + use_fp8_w8a8: bool,
213 + use_int8_w8a16: bool,
214 + search_space: List[Dict[str, int]],
215 + ) -> Dict[str, int]:
216 best_config = None
217 best_time = float("inf")
218 for config in tqdm(search_space):
@@ -198,7 +224,8 @@ class BenchmarkWorker:
224 hidden_size,
225 topk,
226 dtype,
201 - use_fp8,
227 + use_fp8_w8a8,
228 + use_int8_w8a16,
229 num_iters=10)
230 except triton.runtime.autotuner.OutOfResources:
231 # Some configurations may be invalid and fail to compile.
@@ -224,20 +251,19 @@ def sort_config(config: BenchmarkConfig) -> BenchmarkConfig:
251 }
252
253
227 -def save_configs(
228 - configs: Dict[int, BenchmarkConfig],
229 - num_experts: int,
230 - shard_intermediate_size: int,
231 - hidden_size: int,
232 - topk: int,
233 - dtype: torch.dtype,
234 - use_fp8: bool,
235 -) -> None:
236 - dtype_str = "float8" if use_fp8 else None
254 +def save_configs(configs: Dict[int, BenchmarkConfig], num_experts: int,
255 + shard_intermediate_size: int, hidden_size: int, topk: int,
256 + dtype: torch.dtype, use_fp8_w8a8: bool,
257 + use_int8_w8a16: bool) -> None:
258 + dtype_str = get_config_dtype_str(dtype,
259 + use_int8_w8a16=use_int8_w8a16,
260 + use_fp8_w8a8=use_fp8_w8a8)
261 +
262 # NOTE(woosuk): The current naming convention uses w2.shape[2], which
263 # is the intermediate size after silu_and_mul.
264 filename = get_config_file_name(num_experts, shard_intermediate_size // 2,
265 dtype_str)
266 +
267 print(f"Writing best config to {filename}...")
268 with open(filename, "w") as f:
269 json.dump(configs, f, indent=4)
@@ -253,6 +279,11 @@ def main(args: argparse.Namespace):
279 topk = config.ffn_config.moe_top_k
280 intermediate_size = config.ffn_config.ffn_hidden_size
281 shard_intermediate_size = 2 * intermediate_size // args.tp_size
282 + elif config.architectures[0] == "JambaForCausalLM":
283 + E = config.num_experts
284 + topk = config.num_experts_per_tok
285 + intermediate_size = config.intermediate_size
286 + shard_intermediate_size = 2 * intermediate_size // args.tp_size
287 else:
288 # Default: Mixtral.
289 E = config.num_local_experts
@@ -262,7 +293,8 @@ def main(args: argparse.Namespace):
293
294 hidden_size = config.hidden_size
295 dtype = config.torch_dtype
265 - use_fp8 = args.dtype == "fp8"
296 + use_fp8_w8a8 = args.dtype == "fp8_w8a8"
297 + use_int8_w8a16 = args.dtype == "int8_w8a16"
298
299 if args.batch_size is None:
300 batch_sizes = [
@@ -294,21 +326,21 @@ def main(args: argparse.Namespace):
326 start = time.time()
327 configs = _distribute(
328 "tune", [(batch_size, E, shard_intermediate_size, hidden_size,
297 - topk, dtype, use_fp8, search_space)
329 + topk, dtype, use_fp8_w8a8, use_int8_w8a16, search_space)
330 for batch_size in batch_sizes])
331 best_configs = {
332 M: sort_config(config)
333 for M, config in zip(batch_sizes, configs)
334 }
335 save_configs(best_configs, E, shard_intermediate_size, hidden_size,
304 - topk, dtype, use_fp8)
336 + topk, dtype, use_fp8_w8a8, use_int8_w8a16)
337 end = time.time()
338 print(f"Tuning took {end - start:.2f} seconds")
339 else:
308 - outputs = _distribute("benchmark",
309 - [(batch_size, E, shard_intermediate_size,
310 - hidden_size, topk, dtype, use_fp8)
311 - for batch_size in batch_sizes])
340 + outputs = _distribute(
341 + "benchmark", [(batch_size, E, shard_intermediate_size, hidden_size,
342 + topk, dtype, use_fp8_w8a8, use_int8_w8a16)
343 + for batch_size in batch_sizes])
344
345 for batch_size, (config, kernel_time) in zip(batch_sizes, outputs):
346 print(f"Batch size: {batch_size}, config: {config}")
@@ -323,7 +355,7 @@ if __name__ == "__main__":
355 parser.add_argument("--tp-size", "-tp", type=int, default=2)
356 parser.add_argument("--dtype",
357 type=str,
326 - choices=["auto", "fp8"],
358 + choices=["auto", "fp8_w8a8", "int8_w8a16"],
359 default="auto")
360 parser.add_argument("--seed", type=int, default=0)
361 parser.add_argument("--batch-size", type=int, required=False)
tests/models/test_jamba.py
+7 -6
@@ -6,9 +6,12 @@ from vllm.worker.model_runner import _get_graph_batch_size
6 MODELS = ["ai21labs/Jamba-tiny-random"]
7
8
9 +# Fails due to usage of MoE as MLP(E=1_, which is different than the HF impl
10 +# TODO: Fix this with trained model
11 +@pytest.mark.skip()
12 @pytest.mark.parametrize("model", MODELS)
10 -@pytest.mark.parametrize("dtype", ["float"])
11 -@pytest.mark.parametrize("max_tokens", [20])
13 +@pytest.mark.parametrize("dtype", ["bfloat16"])
14 +@pytest.mark.parametrize("max_tokens", [10])
15 def test_models(
16 hf_runner,
17 vllm_runner,
@@ -17,8 +20,6 @@ def test_models(
20 dtype: str,
21 max_tokens: int,
22 ) -> None:
20 - # To pass the small model tests, we need full precision.
21 - assert dtype == "float"
23
24 with hf_runner(model, dtype=dtype) as hf_model:
25 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
@@ -36,8 +37,8 @@ def test_models(
37
38
39 @pytest.mark.parametrize("model", MODELS)
39 -@pytest.mark.parametrize("dtype", ["float"])
40 -@pytest.mark.parametrize("max_tokens", [20])
40 +@pytest.mark.parametrize("dtype", ["half"])
41 +@pytest.mark.parametrize("max_tokens", [5])
42 def test_batching(
43 vllm_runner,
44 example_prompts,
tests/quantization/test_experts_int8.py new
+28
@@ -0,0 +1,28 @@
1 +# flake8: noqa
2 +"""Tests experts_int8 quantization startup and generation,
3 +doesn't test correctness
4 +"""
5 +import pytest
6 +
7 +from tests.quantization.utils import is_quant_method_supported
8 +
9 +MODELS = ["ai21labs/Jamba-tiny-random"]
10 +
11 +
12 +@pytest.mark.skipif(not is_quant_method_supported("experts_int8"),
13 + reason="ExpertsInt8 is not supported on this GPU type.")
14 +@pytest.mark.parametrize("model", MODELS)
15 +@pytest.mark.parametrize("dtype", ["bfloat16"])
16 +@pytest.mark.parametrize("max_tokens", [10])
17 +def test_model_experts_int8_startup(
18 + hf_runner,
19 + vllm_runner,
20 + example_prompts,
21 + model: str,
22 + dtype: str,
23 + max_tokens: int,
24 +) -> None:
25 +
26 + with vllm_runner(model, dtype=dtype,
27 + quantization="experts_int8") as vllm_model:
28 + vllm_model.generate_greedy(example_prompts, max_tokens)
vllm/config.py
+2 -1
@@ -243,7 +243,8 @@ class ModelConfig:
243 rocm_supported_quantization = ["gptq", "squeezellm", "fp8"]
244 optimized_quantization_methods = [
245 "fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
246 - "fbgemm_fp8", "compressed_tensors", "compressed-tensors"
246 + "fbgemm_fp8", "compressed_tensors", "compressed-tensors",
247 + "experts_int8"
248 ]
249 tpu_supported_quantization = ["tpu_int8"]
250 if self.quantization is not None:
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json renamed
vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json renamed
vllm/model_executor/layers/fused_moe/fused_moe.py
+91 -53
@@ -17,42 +17,44 @@ logger = init_logger(__name__)
17
18 @triton.jit
19 def fused_moe_kernel(
20 - # Pointers to matrices
21 - a_ptr,
22 - b_ptr,
23 - c_ptr,
24 - a_scale_ptr,
25 - b_scale_ptr,
26 - topk_weights_ptr,
27 - sorted_token_ids_ptr,
28 - expert_ids_ptr,
29 - num_tokens_post_padded_ptr,
30 - # Matrix dimensions
31 - N,
32 - K,
33 - EM,
34 - num_valid_tokens,
35 - # The stride variables represent how much to increase the ptr by when
36 - # moving by 1 element in a particular dimension. E.g. `stride_am` is
37 - # how much to increase `a_ptr` by to get the element one row down
38 - # (A has M rows).
39 - stride_am,
40 - stride_ak,
41 - stride_be,
42 - stride_bk,
43 - stride_bn,
44 - stride_cm,
45 - stride_cn,
46 - # Meta-parameters
47 - BLOCK_SIZE_M: tl.constexpr,
48 - BLOCK_SIZE_N: tl.constexpr,
49 - BLOCK_SIZE_K: tl.constexpr,
50 - GROUP_SIZE_M: tl.constexpr,
51 - MUL_ROUTED_WEIGHT: tl.constexpr,
52 - top_k: tl.constexpr,
53 - compute_type: tl.constexpr,
54 - use_fp8: tl.constexpr,
55 -):
20 + # Pointers to matrices
21 + a_ptr,
22 + b_ptr,
23 + c_ptr,
24 + a_scale_ptr,
25 + b_scale_ptr,
26 + topk_weights_ptr,
27 + sorted_token_ids_ptr,
28 + expert_ids_ptr,
29 + num_tokens_post_padded_ptr,
30 + # Matrix dimensions
31 + N,
32 + K,
33 + EM,
34 + num_valid_tokens,
35 + # The stride variables represent how much to increase the ptr by when
36 + # moving by 1 element in a particular dimension. E.g. `stride_am` is
37 + # how much to increase `a_ptr` by to get the element one row down
38 + # (A has M rows).
39 + stride_am,
40 + stride_ak,
41 + stride_be,
42 + stride_bk,
43 + stride_bn,
44 + stride_cm,
45 + stride_cn,
46 + stride_bse,
47 + stride_bsn,
48 + # Meta-parameters
49 + BLOCK_SIZE_M: tl.constexpr,
50 + BLOCK_SIZE_N: tl.constexpr,
51 + BLOCK_SIZE_K: tl.constexpr,
52 + GROUP_SIZE_M: tl.constexpr,
53 + MUL_ROUTED_WEIGHT: tl.constexpr,
54 + top_k: tl.constexpr,
55 + compute_type: tl.constexpr,
56 + use_fp8_w8a8: tl.constexpr,
57 + use_int8_w8a16: tl.constexpr):
58 """
59 Implements the fused computation for a Mixture of Experts (MOE) using
60 token and expert matrices.
@@ -113,8 +115,12 @@ def fused_moe_kernel(
115 off_experts = tl.load(expert_ids_ptr + pid_m)
116 b_ptrs = b_ptr + off_experts * stride_be + (offs_k[:, None] * stride_bk +
117 offs_bn[None, :] * stride_bn)
118 + if use_int8_w8a16:
119 + b_scale_ptrs = b_scale_ptr + off_experts * stride_bse + offs_bn[
120 + None, :] * stride_bsn
121 + b_scale = tl.load(b_scale_ptrs)
122
117 - if use_fp8:
123 + if use_fp8_w8a8:
124 a_scale = tl.load(a_scale_ptr)
125 b_scale = tl.load(b_scale_ptr + off_experts)
126
@@ -136,7 +142,9 @@ def fused_moe_kernel(
142 mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
143 other=0.0)
144 # We accumulate along the K dimension.
139 - if use_fp8:
145 + if use_int8_w8a16:
146 + accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
147 + elif use_fp8_w8a8:
148 accumulator = tl.dot(a, b, acc=accumulator)
149 else:
150 accumulator += tl.dot(a, b)
@@ -149,8 +157,9 @@ def fused_moe_kernel(
157 mask=token_mask,
158 other=0)
159 accumulator = accumulator * moe_weight[:, None]
152 -
153 - if use_fp8:
160 + if use_int8_w8a16:
161 + accumulator = (accumulator * b_scale).to(compute_type)
162 + elif use_fp8_w8a8:
163 accumulator = (accumulator * a_scale * b_scale).to(compute_type)
164 else:
165 accumulator = accumulator.to(compute_type)
@@ -229,16 +238,18 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
238 num_tokens_post_padded: torch.Tensor,
239 mul_routed_weight: bool, top_k: int,
240 config: Dict[str, Any], compute_type: tl.dtype,
232 - use_fp8: bool) -> None:
241 + use_fp8_w8a8: bool, use_int8_w8a16: bool) -> None:
242 assert topk_weights.stride(1) == 1
243 assert sorted_token_ids.stride(0) == 1
244
236 - if not use_fp8:
237 - assert A_scale is None
238 - assert B_scale is None
239 - else:
245 + if use_fp8_w8a8:
246 A, A_scale = ops.scaled_fp8_quant(A, A_scale)
247 assert B_scale is not None
248 + elif use_int8_w8a16:
249 + assert B_scale is not None
250 + else:
251 + assert A_scale is None
252 + assert B_scale is None
253
254 grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[
255 'BLOCK_SIZE_M']) * triton.cdiv(B.shape[1], META['BLOCK_SIZE_N']), )
@@ -264,10 +275,13 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
275 B.stride(1),
276 C.stride(1),
277 C.stride(2),
278 + B_scale.stride(0) if B_scale is not None and use_int8_w8a16 else 0,
279 + B_scale.stride(1) if B_scale is not None and use_int8_w8a16 else 0,
280 MUL_ROUTED_WEIGHT=mul_routed_weight,
281 top_k=top_k,
282 compute_type=compute_type,
270 - use_fp8=use_fp8,
283 + use_fp8_w8a8=use_fp8_w8a8,
284 + use_int8_w8a16=use_int8_w8a16,
285 **config,
286 )
287
@@ -426,6 +440,20 @@ def grouped_topk(hidden_states: torch.Tensor,
440 return topk_weights, topk_ids
441
442
443 +def get_config_dtype_str(dtype: torch.dtype,
444 + use_int8_w8a16: Optional[bool] = False,
445 + use_fp8_w8a8: Optional[bool] = False):
446 + if use_fp8_w8a8:
447 + return "fp8_w8a8"
448 + elif use_int8_w8a16:
449 + return "int8_w8a16"
450 + elif dtype == torch.float:
451 + # avoiding cases where kernel fails when float32 MoE
452 + # use fp16/bfloat16 configs
453 + return "float32"
454 + return None
455 +
456 +
457 def fused_experts(hidden_states: torch.Tensor,
458 w1: torch.Tensor,
459 w2: torch.Tensor,
@@ -433,7 +461,8 @@ def fused_experts(hidden_states: torch.Tensor,
461 topk_ids: torch.Tensor,
462 inplace: bool = False,
463 override_config: Optional[Dict[str, Any]] = None,
436 - use_fp8: bool = False,
464 + use_fp8_w8a8: bool = False,
465 + use_int8_w8a16: bool = False,
466 w1_scale: Optional[torch.Tensor] = None,
467 w2_scale: Optional[torch.Tensor] = None,
468 a1_scale: Optional[torch.Tensor] = None,
@@ -454,13 +483,16 @@ def fused_experts(hidden_states: torch.Tensor,
483 # https://github.com/vllm-project/vllm/issues/5938
484 CHUNK_SIZE = envs.VLLM_FUSED_MOE_CHUNK_SIZE
485 M = min(num_tokens, CHUNK_SIZE)
486 + config_dtype = get_config_dtype_str(use_fp8_w8a8=use_fp8_w8a8,
487 + use_int8_w8a16=use_int8_w8a16,
488 + dtype=hidden_states.dtype)
489
490 get_config_func = functools.partial(
491 try_get_optimal_moe_config,
492 w1.shape,
493 w2.shape,
494 topk_ids.shape[1],
463 - "float8" if use_fp8 else None,
495 + config_dtype,
496 override_config=override_config,
497 )
498
@@ -524,7 +556,8 @@ def fused_experts(hidden_states: torch.Tensor,
556 topk_ids.shape[1],
557 config,
558 compute_type=compute_type,
527 - use_fp8=use_fp8)
559 + use_fp8_w8a8=use_fp8_w8a8,
560 + use_int8_w8a16=use_int8_w8a16)
561
562 ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
563
@@ -542,7 +575,8 @@ def fused_experts(hidden_states: torch.Tensor,
575 1,
576 config,
577 compute_type=compute_type,
545 - use_fp8=use_fp8)
578 + use_fp8_w8a8=use_fp8_w8a8,
579 + use_int8_w8a16=use_int8_w8a16)
580
581 torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
582 dim=1,
@@ -562,7 +596,8 @@ def fused_moe(
596 use_grouped_topk: bool = False,
597 num_expert_group: Optional[int] = None,
598 topk_group: Optional[int] = None,
565 - use_fp8: bool = False,
599 + use_fp8_w8a8: bool = False,
600 + use_int8_w8a16: bool = False,
601 w1_scale: Optional[torch.Tensor] = None,
602 w2_scale: Optional[torch.Tensor] = None,
603 a1_scale: Optional[torch.Tensor] = None,
@@ -588,7 +623,9 @@ def fused_moe(
623 - topk_group: Optional[int]: additional parameter for grouped_topk
624 - use_grouped_topk: If True, use grouped_topk instead of fused_topk
625 note: Deepseekv2 model uses grouped_topk
591 - - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
626 + - use_fp8_w8a8 (bool): If True, use fp8 arithmetic to compute the inner
627 + products for w1 and w2. Defaults to False.
628 + - use_int8_w8a16 (bool): If True, use fp8 arithmetic to compute the inner
629 products for w1 and w2. Defaults to False.
630 - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
631 w1.
@@ -617,7 +654,8 @@ def fused_moe(
654 topk_ids,
655 inplace=inplace,
656 override_config=override_config,
620 - use_fp8=use_fp8,
657 + use_fp8_w8a8=use_fp8_w8a8,
658 + use_int8_w8a16=use_int8_w8a16,
659 w1_scale=w1_scale,
660 w2_scale=w2_scale,
661 a1_scale=a1_scale,
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -11,6 +11,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
11 CompressedTensorsConfig)
12 from vllm.model_executor.layers.quantization.deepspeedfp import (
13 DeepSpeedFPConfig)
14 +from vllm.model_executor.layers.quantization.experts_int8 import (
15 + ExpertsInt8Config)
16 from vllm.model_executor.layers.quantization.fbgemm_fp8 import FBGEMMFp8Config
17 from vllm.model_executor.layers.quantization.fp8 import Fp8Config
18 from vllm.model_executor.layers.quantization.gguf import GGUFConfig
@@ -43,6 +45,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
45 "compressed-tensors": CompressedTensorsConfig,
46 "bitsandbytes": BitsAndBytesConfig,
47 "qqq": QQQConfig,
48 + "experts_int8": ExpertsInt8Config,
49 }
50
51
vllm/model_executor/layers/quantization/experts_int8.py new
+175
@@ -0,0 +1,175 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +
5 +from vllm.distributed import get_tensor_model_parallel_rank, get_tp_group
6 +from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
7 +from vllm.model_executor.layers.linear import (LinearBase,
8 + UnquantizedLinearMethod)
9 +from vllm.model_executor.layers.quantization.base_config import (
10 + QuantizationConfig, QuantizeMethodBase)
11 +from vllm.model_executor.utils import set_weight_attrs
12 +
13 +
14 +class ExpertsInt8Config(QuantizationConfig):
15 + """Config class for Int8 experts quantization."""
16 +
17 + def __init__(self) -> None:
18 + pass
19 +
20 + @classmethod
21 + def get_name(cls) -> str:
22 + return "experts_int8"
23 +
24 + @classmethod
25 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
26 + return [torch.bfloat16, torch.half]
27 +
28 + @classmethod
29 + def get_min_capability(cls) -> int:
30 + return 80
31 +
32 + @classmethod
33 + def get_config_filenames(cls) -> List[str]:
34 + return []
35 +
36 + @classmethod
37 + def from_config(cls, config: Dict[str, Any]) -> "ExpertsInt8Config":
38 + return cls()
39 +
40 + def get_quant_method(self, layer: torch.nn.Module,
41 + prefix: str) -> Optional["QuantizeMethodBase"]:
42 + if isinstance(layer, LinearBase):
43 + return UnquantizedLinearMethod()
44 + elif isinstance(layer, FusedMoE):
45 + return ExpertsInt8MoEMethod(self)
46 + return None
47 +
48 + def get_scaled_act_names(self) -> List[str]:
49 + return []
50 +
51 +
52 +class ExpertsInt8MoEMethod(FusedMoEMethodBase):
53 +
54 + def __init__(self, quant_config: ExpertsInt8Config):
55 + self.quant_config = quant_config
56 +
57 + def create_weights(self, layer: torch.nn.Module, num_experts: int,
58 + hidden_size: int, intermediate_size: int,
59 + params_dtype: torch.dtype, **extra_weight_attrs):
60 +
61 + int8_dtype = torch.int8
62 +
63 + assert 'weight_loader' in extra_weight_attrs
64 + weight_loader = extra_weight_attrs['weight_loader']
65 + wrapped_weight_loader = ExpertsInt8MoEMethod.quantizing_weight_loader(
66 + layer, weight_loader)
67 + extra_weight_attrs['weight_loader'] = wrapped_weight_loader
68 +
69 + # Fused gate_up_proj (column parallel)
70 + w13_weight = torch.nn.Parameter(torch.empty(num_experts,
71 + 2 * intermediate_size,
72 + hidden_size,
73 + dtype=int8_dtype),
74 + requires_grad=False)
75 + layer.register_parameter("w13_weight", w13_weight)
76 + set_weight_attrs(w13_weight, extra_weight_attrs)
77 +
78 + # down_proj (row parallel)
79 + w2_weight = torch.nn.Parameter(torch.empty(num_experts,
80 + hidden_size,
81 + intermediate_size,
82 + dtype=int8_dtype),
83 + requires_grad=False)
84 + layer.register_parameter("w2_weight", w2_weight)
85 + set_weight_attrs(w2_weight, extra_weight_attrs)
86 +
87 + w13_scale = torch.nn.Parameter(torch.zeros(num_experts,
88 + 2 * intermediate_size,
89 + dtype=torch.float32),
90 + requires_grad=False)
91 + layer.register_parameter("w13_scale", w13_scale)
92 +
93 + w2_scale = torch.nn.Parameter(torch.zeros(num_experts,
94 + hidden_size,
95 + dtype=torch.float32),
96 + requires_grad=False)
97 + layer.register_parameter("w2_scale", w2_scale)
98 +
99 + def apply(self,
100 + layer: torch.nn.Module,
101 + x: torch.Tensor,
102 + router_logits: torch.Tensor,
103 + top_k: int,
104 + renormalize: bool = True,
105 + use_grouped_topk: bool = False,
106 + num_expert_group: Optional[int] = None,
107 + topk_group: Optional[int] = None) -> torch.Tensor:
108 + from vllm.model_executor.layers.fused_moe import fused_experts
109 +
110 + topk_weights, topk_ids = FusedMoE.select_experts(
111 + hidden_states=x,
112 + router_logits=router_logits,
113 + use_grouped_topk=use_grouped_topk,
114 + top_k=top_k,
115 + renormalize=renormalize,
116 + topk_group=topk_group,
117 + num_expert_group=num_expert_group)
118 +
119 + return fused_experts(x,
120 + layer.w13_weight,
121 + layer.w2_weight,
122 + topk_weights=topk_weights,
123 + topk_ids=topk_ids,
124 + inplace=True,
125 + use_int8_w8a16=True,
126 + w1_scale=layer.w13_scale,
127 + w2_scale=layer.w2_scale)
128 +
129 + @staticmethod
130 + def quantizing_weight_loader(layer, weight_loader):
131 +
132 + def quantize_and_call_weight_loader(param: torch.nn.Parameter,
133 + loaded_weight: torch.Tensor,
134 + weight_name: str, shard_id: int,
135 + expert_id: int):
136 + tp_rank = get_tensor_model_parallel_rank()
137 + shard_size = layer.intermediate_size_per_partition
138 + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
139 + device = get_tp_group().device
140 + loaded_weight = loaded_weight.to(device)
141 + # w1, gate_proj case: Load into first shard of w13.
142 + if shard_id == "w1":
143 + scales = quantize_in_place_and_get_scales(
144 + loaded_weight[shard, :])
145 + layer.w13_scale.data[expert_id, 0:shard_size].copy_(scales[:,
146 + 0])
147 + # w3, up_proj case: Load into second shard of w13.
148 + elif shard_id == "w3":
149 + scales = quantize_in_place_and_get_scales(
150 + loaded_weight[shard, :])
151 + layer.w13_scale.data[expert_id, shard_size:2 *
152 + shard_size].copy_(scales[:, 0])
153 + # w2, down_proj case: Load into only shard of w2.
154 + elif shard_id == "w2":
155 + scales = quantize_in_place_and_get_scales(loaded_weight[:,
156 + shard])
157 + layer.w2_scale.data[expert_id, :].copy_(scales[:, 0])
158 + else:
159 + raise ValueError(
160 + f"Shard id must be in [0,1,2] but got {shard_id}")
161 + weight_loader(param, loaded_weight, weight_name, shard_id,
162 + expert_id)
163 +
164 + return quantize_and_call_weight_loader
165 +
166 +
167 +def quantize_in_place_and_get_scales(weight: torch.Tensor) -> torch.Tensor:
168 + vmax = torch.iinfo(torch.int8).max
169 + scales = (torch.max(torch.abs(weight), dim=1, keepdim=True)[0] / vmax)
170 +
171 + weight.div_(scales)
172 + weight.round_()
173 + weight.clamp_(-vmax, vmax)
174 +
175 + return scales
vllm/model_executor/layers/quantization/fp8.py
+1 -1
@@ -488,7 +488,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
488 topk_weights=topk_weights,
489 topk_ids=topk_ids,
490 inplace=True,
491 - use_fp8=True,
491 + use_fp8_w8a8=True,
492 w1_scale=layer.w13_weight_scale,
493 w2_scale=layer.w2_weight_scale,
494 a1_scale=layer.w13_input_scale,
vllm/model_executor/models/jamba.py
+35 -37
@@ -16,7 +16,6 @@ from vllm.attention.layer import Attention
16 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
17 from vllm.distributed import (get_tensor_model_parallel_rank,
18 get_tensor_model_parallel_world_size)
19 -from vllm.model_executor.layers.activation import SiluAndMul
19 from vllm.model_executor.layers.fused_moe import FusedMoE
20 from vllm.model_executor.layers.layernorm import RMSNorm
21 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -249,37 +248,6 @@ class JambaMambaMixer(nn.Module):
248 return hidden_states
249
250
252 -class JambaMLP(nn.Module):
253 -
254 - def __init__(
255 - self,
256 - config: JambaConfig,
257 - quant_config: Optional[QuantizationConfig] = None,
258 - ) -> None:
259 - super().__init__()
260 - hidden_size = config.hidden_size
261 - intermediate_size = config.intermediate_size
262 - hidden_act = config.hidden_act
263 - self.gate_up_proj = MergedColumnParallelLinear(
264 - hidden_size, [intermediate_size] * 2,
265 - bias=False,
266 - quant_config=quant_config)
267 - self.down_proj = RowParallelLinear(intermediate_size,
268 - hidden_size,
269 - bias=False,
270 - quant_config=quant_config)
271 - if hidden_act != "silu":
272 - raise ValueError(f"Unsupported activation: {hidden_act}. "
273 - "Only silu is supported for now.")
274 - self.act_fn = SiluAndMul()
275 -
276 - def forward(self, x):
277 - gate_up, _ = self.gate_up_proj(x)
278 - x = self.act_fn(gate_up)
279 - x, _ = self.down_proj(x)
280 - return x
281 -
282 -
251 class JambaMoE(nn.Module):
252
253 def __init__(self,
@@ -327,6 +295,21 @@ class JambaMoE(nn.Module):
295 return hidden_states.view(orig_shape)
296
297
298 +class JambaMLP(JambaMoE):
299 +
300 + def __init__(self,
301 + config: JambaConfig,
302 + params_dtype: Optional[torch.dtype] = None,
303 + tp_size: Optional[int] = None,
304 + quant_config: Optional[QuantizationConfig] = None):
305 + super().__init__(config,
306 + num_experts=1,
307 + top_k=1,
308 + params_dtype=params_dtype,
309 + tp_size=tp_size,
310 + quant_config=quant_config)
311 +
312 +
313 class JambaMambaDecoderLayer(nn.Module):
314
315 def __init__(self,
@@ -884,8 +867,6 @@ class JambaForCausalLM(nn.Module, HasInnerState):
867 ("qkv_proj", "q_proj", "q"),
868 ("qkv_proj", "k_proj", "k"),
869 ("qkv_proj", "v_proj", "v"),
887 - ("gate_up_proj", "gate_proj", 0),
888 - ("gate_up_proj", "up_proj", 1),
870 ]
871
872 # Params for weights, fp8 weight scales, fp8 activation scales
@@ -907,6 +888,10 @@ class JambaForCausalLM(nn.Module, HasInnerState):
888 if ".self_attn." in name:
889 name = name.replace(".self_attn", "")
890
891 + if "feed_forward" in name and not _is_moe_layer(name):
892 + ## map MLP layers to expert with ID=0
893 + name = name.replace("feed_forward", "feed_forward.experts.0")
894 +
895 for param_name, weight_name, shard_id in stacked_params_mapping:
896 if weight_name not in name:
897 continue
@@ -921,16 +906,21 @@ class JambaForCausalLM(nn.Module, HasInnerState):
906 weight_loader(param, loaded_weight, shard_id)
907 break
908 else:
924 - for mapping in expert_params_mapping:
925 - param_name, weight_name, expert_id, shard_id = mapping
909 + for (
910 + param_name,
911 + weight_name,
912 + expert_id,
913 + shard_id,
914 + ) in expert_params_mapping:
915 if weight_name not in name:
916 continue
917 +
918 name = name.replace(weight_name, param_name)
919 param = params_dict[name]
920 weight_loader = param.weight_loader
921 weight_loader(param,
922 loaded_weight,
933 - name,
923 + weight_name,
924 shard_id=shard_id,
925 expert_id=expert_id)
926 break
@@ -943,3 +933,11 @@ class JambaForCausalLM(nn.Module, HasInnerState):
933 weight_loader = getattr(param, "weight_loader",
934 default_weight_loader)
935 weight_loader(param, loaded_weight)
936 +
937 +
938 +def _is_moe_layer(name: str):
939 + return any(
940 + [experts_name in name for experts_name in [
941 + "experts",
942 + "router",
943 + ]])