@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1248e8506a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Model] Adding support for MSFT Phi-3.5-MoE (#7729)

Co-authored-by: Your Name <you@example.com> Co-authored-by: Zeqi Lin <zelin@microsoft.com> Co-authored-by: Zeqi Lin <Zeqi.Lin@microsoft.com>

Wenxiang committed Aug 31, 2024 at 03:42 UTC 1248e8506a4d98b4f15cbfe729cf2af42fb4223a
13 files changed +1255 -82
docs/source/models/supported_models.rst
+4
@@ -147,6 +147,10 @@ Decoder-only Language Models
147 - Phi-3-Small
148 - :code:`microsoft/Phi-3-small-8k-instruct`, :code:`microsoft/Phi-3-small-128k-instruct`, etc.
149 -
150 + * - :code:`PhiMoEForCausalLM`
151 + - Phi-3.5-MoE
152 + - :code:`microsoft/Phi-3.5-MoE-instruct`, etc.
153 + -
154 * - :code:`PersimmonForCausalLM`
155 - Persimmon
156 - :code:`adept/persimmon-8b-base`, :code:`adept/persimmon-8b-chat`, etc.
tests/models/test_phimoe.py new
+111
@@ -0,0 +1,111 @@
1 +"""Compare the outputs of HF and vLLM for moe models using greedy sampling.
2 +
3 +Run `pytest tests/models/test_phimoe.py`.
4 +"""
5 +import pytest
6 +import torch
7 +
8 +from vllm.utils import is_cpu
9 +
10 +from .utils import check_logprobs_close
11 +
12 +MODELS = [
13 + "microsoft/Phi-3.5-MoE-instruct",
14 +]
15 +
16 +
17 +def test_phimoe_routing_function():
18 + from vllm.model_executor.models.phimoe import phimoe_routing_function
19 + test_case = {
20 + 0: {
21 + "hidden_states":
22 + torch.tensor([1, 2, 3, 4, 5, 6, 7, 8],
23 + dtype=torch.float32,
24 + requires_grad=False).view(4, 2),
25 + "gating_output":
26 + torch.tensor([0.1, 0.2, 0.3, 0.4],
27 + dtype=torch.float32,
28 + requires_grad=False),
29 + "topk":
30 + 2,
31 + "renormalize":
32 + False,
33 + },
34 + 1: {
35 + "hidden_states":
36 + torch.tensor([1, 2, 3, 4, 5, 6, 7, 8],
37 + dtype=torch.float32,
38 + requires_grad=False).view(4, 2),
39 + "gating_output":
40 + torch.tensor([0.4, 0.2, 0.3, 0.4],
41 + dtype=torch.float32,
42 + requires_grad=False),
43 + "topk":
44 + 2,
45 + "renormalize":
46 + False,
47 + }
48 + }
49 +
50 + ground_truth = {
51 + 0: {
52 + "topk_weights":
53 + torch.tensor([1., 1.], dtype=torch.float32, requires_grad=False),
54 + "topk_ids":
55 + torch.tensor([3, 2], dtype=torch.long, requires_grad=False),
56 + },
57 + 1: {
58 + "topk_weights":
59 + torch.tensor([0.5, 1.], dtype=torch.float32, requires_grad=False),
60 + "topk_ids":
61 + torch.tensor([0, 3], dtype=torch.long, requires_grad=False),
62 + }
63 + }
64 +
65 + for test_id in test_case:
66 + topk_weights, topk_ids = phimoe_routing_function(**test_case[test_id])
67 + assert torch.allclose(topk_weights,
68 + ground_truth[test_id]["topk_weights"])
69 + assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
70 +
71 +
72 +def get_gpu_memory():
73 + try:
74 + props = torch.cuda.get_device_properties(torch.cuda.current_device())
75 + gpu_memory = props.total_memory / (1024**3)
76 + return gpu_memory
77 + except Exception:
78 + return 0
79 +
80 +
81 +@pytest.mark.skipif(condition=is_cpu(),
82 + reason="This test takes a lot time to run on CPU, "
83 + "and vllm CI's disk space is not enough for this model.")
84 +@pytest.mark.skipif(condition=get_gpu_memory() < 100,
85 + reason="Skip this test if GPU memory is insufficient.")
86 +@pytest.mark.parametrize("model", MODELS)
87 +@pytest.mark.parametrize("dtype", ["bfloat16"])
88 +@pytest.mark.parametrize("max_tokens", [64])
89 +@pytest.mark.parametrize("num_logprobs", [5])
90 +def test_models(
91 + hf_runner,
92 + vllm_runner,
93 + example_prompts,
94 + model: str,
95 + dtype: str,
96 + max_tokens: int,
97 + num_logprobs: int,
98 +) -> None:
99 + with hf_runner(model, dtype=dtype) as hf_model:
100 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
101 + example_prompts, max_tokens, num_logprobs)
102 +
103 + with vllm_runner(model, dtype=dtype) as vllm_model:
104 + vllm_outputs = vllm_model.generate_greedy_logprobs(
105 + example_prompts, max_tokens, num_logprobs)
106 + check_logprobs_close(
107 + outputs_0_lst=hf_outputs,
108 + outputs_1_lst=vllm_outputs,
109 + name_0="hf",
110 + name_1="vllm",
111 + )
vllm/model_executor/layers/fused_moe/configs/E=16,N=3200,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new
+130
@@ -0,0 +1,130 @@
1 +{
2 + "3328": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 256,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 2
9 + },
10 + "1024": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 32,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "3072": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 2
25 + },
26 + "256": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 256,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 8,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "768": {
35 + "BLOCK_SIZE_M": 128,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 8,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "1792": {
43 + "BLOCK_SIZE_M": 128,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "2560": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 256,
53 + "BLOCK_SIZE_K": 64,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "2816": {
59 + "BLOCK_SIZE_M": 128,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "3584": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 256,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "1536": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 256,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "2048": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 256,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "512": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 8,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "3840": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1280": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "2304": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "4096": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 2
129 + }
130 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=16,N=6400,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new
+130
@@ -0,0 +1,130 @@
1 +{
2 + "3840": {
3 + "BLOCK_SIZE_M": 128,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 8,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "1792": {
11 + "BLOCK_SIZE_M": 128,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 8,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "3584": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 2
25 + },
26 + "512": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 256,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 2
33 + },
34 + "3072": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 2
41 + },
42 + "2048": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 256,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "2816": {
51 + "BLOCK_SIZE_M": 128,
52 + "BLOCK_SIZE_N": 256,
53 + "BLOCK_SIZE_K": 32,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 8,
56 + "num_stages": 4
57 + },
58 + "1280": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 256,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "768": {
67 + "BLOCK_SIZE_M": 128,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "4096": {
75 + "BLOCK_SIZE_M": 128,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 8,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "3328": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 256,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "2560": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 8,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "1024": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 32,
102 + "GROUP_SIZE_M": 8,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "2304": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "256": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 4
129 + }
130 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=16,N=800,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new
+130
@@ -0,0 +1,130 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 256,
5 + "BLOCK_SIZE_K": 32,
6 + "GROUP_SIZE_M": 8,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "1792": {
11 + "BLOCK_SIZE_M": 128,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "512": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 8,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "3328": {
27 + "BLOCK_SIZE_M": 128,
28 + "BLOCK_SIZE_N": 256,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 8,
32 + "num_stages": 2
33 + },
34 + "3072": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 2
41 + },
42 + "2560": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 32,
46 + "GROUP_SIZE_M": 8,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "768": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 256,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "2816": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 256,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "256": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "4096": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 256,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "1024": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 256,
85 + "BLOCK_SIZE_K": 256,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "2304": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 256,
94 + "GROUP_SIZE_M": 8,
95 + "num_warps": 8,
96 + "num_stages": 2
97 + },
98 + "1280": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "3840": {
107 + "BLOCK_SIZE_M": 16,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 256,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 32,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 256,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "3584": {
123 + "BLOCK_SIZE_M": 16,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 32,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + }
130 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/fused_moe.py
+14 -5
@@ -2,7 +2,7 @@
2 import functools
3 import json
4 import os
5 -from typing import Any, Dict, Optional, Tuple
5 +from typing import Any, Callable, Dict, Optional, Tuple
6
7 import torch
8 import triton
@@ -446,7 +446,8 @@ def fused_marlin_moe(hidden_states: torch.Tensor,
446 rand_perm1: torch.Tensor,
447 rand_perm2: torch.Tensor,
448 topk: int,
449 - renormalize: bool,
449 + custom_routing_function: Optional[Callable] = None,
450 + renormalize: bool = True,
451 override_config: Optional[Dict[str, Any]] = None,
452 use_fp8: bool = False,
453 w1_scale: Optional[torch.Tensor] = None,
@@ -497,8 +498,12 @@ def fused_marlin_moe(hidden_states: torch.Tensor,
498 E = w1.shape[0]
499 N = w2.shape[1] * 16
500
500 - topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
501 - renormalize)
501 + if custom_routing_function is None:
502 + topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
503 + renormalize)
504 + else:
505 + topk_weights, topk_ids = custom_routing_function(
506 + hidden_states, gating_output, topk, renormalize)
507
508 get_config_func = functools.partial(try_get_optimal_moe_config,
509 w1.shape,
@@ -695,6 +700,7 @@ def fused_moe(
700 use_grouped_topk: bool = False,
701 num_expert_group: Optional[int] = None,
702 topk_group: Optional[int] = None,
703 + custom_routing_function: Optional[Callable] = None,
704 use_fp8_w8a8: bool = False,
705 use_int8_w8a16: bool = False,
706 w1_scale: Optional[torch.Tensor] = None,
@@ -742,9 +748,12 @@ def fused_moe(
748 topk_weights, topk_ids = grouped_topk(hidden_states, gating_output,
749 topk, renormalize,
750 num_expert_group, topk_group)
745 - else:
751 + elif custom_routing_function is None:
752 topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
753 renormalize)
754 + else:
755 + topk_weights, topk_ids = custom_routing_function(
756 + hidden_states, gating_output, topk, renormalize)
757
758 return fused_experts(hidden_states,
759 w1,
vllm/model_executor/layers/fused_moe/layer.py
+56 -34
@@ -1,6 +1,6 @@
1 from abc import abstractmethod
2 from enum import Enum
3 -from typing import List, Optional, Tuple
3 +from typing import Callable, List, Optional, Tuple
4
5 import torch
6
@@ -62,15 +62,18 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
62 layer.register_parameter("w2_weight", w2_weight)
63 set_weight_attrs(w2_weight, extra_weight_attrs)
64
65 - def apply(self,
66 - layer: torch.nn.Module,
67 - x: torch.Tensor,
68 - router_logits: torch.Tensor,
69 - top_k: int,
70 - renormalize: bool,
71 - use_grouped_topk: bool,
72 - topk_group: Optional[int] = None,
73 - num_expert_group: Optional[int] = None) -> torch.Tensor:
65 + def apply(
66 + self,
67 + layer: torch.nn.Module,
68 + x: torch.Tensor,
69 + router_logits: torch.Tensor,
70 + top_k: int,
71 + renormalize: bool,
72 + use_grouped_topk: bool,
73 + topk_group: Optional[int] = None,
74 + num_expert_group: Optional[int] = None,
75 + custom_routing_function: Optional[Callable] = None
76 + ) -> torch.Tensor:
77
78 return self.forward(x=x,
79 layer=layer,
@@ -79,17 +82,21 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
82 renormalize=renormalize,
83 use_grouped_topk=use_grouped_topk,
84 topk_group=topk_group,
82 - num_expert_group=num_expert_group)
83 -
84 - def forward_cuda(self,
85 - layer: torch.nn.Module,
86 - x: torch.Tensor,
87 - use_grouped_topk: bool,
88 - top_k: int,
89 - router_logits: torch.Tensor,
90 - renormalize: bool,
91 - topk_group: Optional[int] = None,
92 - num_expert_group: Optional[int] = None) -> torch.Tensor:
85 + num_expert_group=num_expert_group,
86 + custom_routing_function=custom_routing_function)
87 +
88 + def forward_cuda(
89 + self,
90 + layer: torch.nn.Module,
91 + x: torch.Tensor,
92 + use_grouped_topk: bool,
93 + top_k: int,
94 + router_logits: torch.Tensor,
95 + renormalize: bool,
96 + topk_group: Optional[int] = None,
97 + num_expert_group: Optional[int] = None,
98 + custom_routing_function: Optional[Callable] = None
99 + ) -> torch.Tensor:
100
101 from vllm.model_executor.layers.fused_moe.fused_moe import (
102 fused_experts)
@@ -101,7 +108,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
108 top_k=top_k,
109 renormalize=renormalize,
110 topk_group=topk_group,
104 - num_expert_group=num_expert_group)
111 + num_expert_group=num_expert_group,
112 + custom_routing_function=custom_routing_function)
113
114 return fused_experts(hidden_states=x,
115 w1=layer.w13_weight,
@@ -114,20 +122,24 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
122 raise NotImplementedError(
123 "The CPU backend currently does not support MoE.")
124
117 - def forward_tpu(self,
118 - layer: torch.nn.Module,
119 - x: torch.Tensor,
120 - use_grouped_topk: bool,
121 - top_k: int,
122 - router_logits: torch.Tensor,
123 - renormalize: bool,
124 - topk_group: Optional[int] = None,
125 - num_expert_group: Optional[int] = None) -> torch.Tensor:
125 + def forward_tpu(
126 + self,
127 + layer: torch.nn.Module,
128 + x: torch.Tensor,
129 + use_grouped_topk: bool,
130 + top_k: int,
131 + router_logits: torch.Tensor,
132 + renormalize: bool,
133 + topk_group: Optional[int] = None,
134 + num_expert_group: Optional[int] = None,
135 + custom_routing_function: Optional[Callable] = None
136 + ) -> torch.Tensor:
137
138 from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe
139 assert not use_grouped_topk
140 assert num_expert_group is None
141 assert topk_group is None
142 + assert custom_routing_function is None
143 return fused_moe(hidden_states=x,
144 w1=layer.w13_weight,
145 w2=layer.w2_weight,
@@ -172,6 +184,7 @@ class FusedMoE(torch.nn.Module):
184 quant_config: Optional[QuantizationConfig] = None,
185 tp_size: Optional[int] = None,
186 prefix: str = "",
187 + custom_routing_function: Optional[Callable] = None,
188 ):
189 super().__init__()
190
@@ -190,6 +203,7 @@ class FusedMoE(torch.nn.Module):
203 assert num_expert_group is not None and topk_group is not None
204 self.num_expert_group = num_expert_group
205 self.topk_group = topk_group
206 + self.custom_routing_function = custom_routing_function
207
208 if quant_config is None:
209 self.quant_method: Optional[QuantizeMethodBase] = (
@@ -390,7 +404,8 @@ class FusedMoE(torch.nn.Module):
404 use_grouped_topk: bool,
405 renormalize: bool,
406 topk_group: Optional[int] = None,
393 - num_expert_group: Optional[int] = None):
407 + num_expert_group: Optional[int] = None,
408 + custom_routing_function: Optional[Callable] = None):
409 from vllm.model_executor.layers.fused_moe.fused_moe import (
410 fused_topk, grouped_topk)
411
@@ -405,11 +420,17 @@ class FusedMoE(torch.nn.Module):
420 renormalize=renormalize,
421 num_expert_group=num_expert_group,
422 topk_group=topk_group)
408 - else:
423 + elif custom_routing_function is None:
424 topk_weights, topk_ids = fused_topk(hidden_states=hidden_states,
425 gating_output=router_logits,
426 topk=top_k,
427 renormalize=renormalize)
428 + else:
429 + topk_weights, topk_ids = custom_routing_function(
430 + hidden_states=hidden_states,
431 + gating_output=router_logits,
432 + topk=top_k,
433 + renormalize=renormalize)
434
435 return topk_weights, topk_ids
436
@@ -426,7 +447,8 @@ class FusedMoE(torch.nn.Module):
447 renormalize=self.renormalize,
448 use_grouped_topk=self.use_grouped_topk,
449 topk_group=self.topk_group,
429 - num_expert_group=self.num_expert_group)
450 + num_expert_group=self.num_expert_group,
451 + custom_routing_function=self.custom_routing_function)
452
453 if self.reduce_results and self.tp_size > 1:
454 final_hidden_states = tensor_model_parallel_all_reduce(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+14 -10
@@ -1,6 +1,6 @@
1 import enum
2 from enum import Enum
3 -from typing import List, Optional
3 +from typing import Callable, List, Optional
4
5 import torch
6
@@ -256,15 +256,18 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
256 )
257 replace_tensor("w2_weight_scale", marlin_w2_scales)
258
259 - def apply(self,
260 - layer: torch.nn.Module,
261 - x: torch.Tensor,
262 - router_logits: torch.Tensor,
263 - top_k: int,
264 - renormalize: bool = True,
265 - use_grouped_topk: bool = False,
266 - num_expert_group: Optional[int] = None,
267 - topk_group: Optional[int] = None) -> torch.Tensor:
259 + def apply(
260 + self,
261 + layer: torch.nn.Module,
262 + x: torch.Tensor,
263 + router_logits: torch.Tensor,
264 + top_k: int,
265 + renormalize: bool = True,
266 + use_grouped_topk: bool = False,
267 + num_expert_group: Optional[int] = None,
268 + topk_group: Optional[int] = None,
269 + custom_routing_function: Optional[Callable] = None,
270 + ) -> torch.Tensor:
271
272 from vllm.model_executor.layers.fused_moe.fused_moe import (
273 fused_marlin_moe)
@@ -278,6 +281,7 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
281 layer.w13_g_idx_sort_indices,
282 layer.w2_g_idx_sort_indices,
283 top_k,
284 + custom_routing_function,
285 renormalize=renormalize,
286 w1_scale=layer.w13_weight_scale,
287 w2_scale=layer.w2_weight_scale)
vllm/model_executor/layers/quantization/experts_int8.py
+15 -11
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional
1 +from typing import Any, Callable, Dict, List, Optional
2
3 import torch
4
@@ -96,15 +96,18 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
96 requires_grad=False)
97 layer.register_parameter("w2_scale", w2_scale)
98
99 - def apply(self,
100 - layer: torch.nn.Module,
101 - x: torch.Tensor,
102 - router_logits: torch.Tensor,
103 - top_k: int,
104 - renormalize: bool = True,
105 - use_grouped_topk: bool = False,
106 - num_expert_group: Optional[int] = None,
107 - topk_group: Optional[int] = None) -> torch.Tensor:
99 + def apply(
100 + self,
101 + layer: torch.nn.Module,
102 + x: torch.Tensor,
103 + router_logits: torch.Tensor,
104 + top_k: int,
105 + renormalize: bool = True,
106 + use_grouped_topk: bool = False,
107 + num_expert_group: Optional[int] = None,
108 + topk_group: Optional[int] = None,
109 + custom_routing_function: Optional[Callable] = None,
110 + ) -> torch.Tensor:
111 from vllm.model_executor.layers.fused_moe import fused_experts
112
113 topk_weights, topk_ids = FusedMoE.select_experts(
@@ -114,7 +117,8 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
117 top_k=top_k,
118 renormalize=renormalize,
119 topk_group=topk_group,
117 - num_expert_group=num_expert_group)
120 + num_expert_group=num_expert_group,
121 + custom_routing_function=custom_routing_function)
122
123 return fused_experts(x,
124 layer.w13_weight,
vllm/model_executor/layers/quantization/fp8.py
+15 -11
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional
1 +from typing import Any, Callable, Dict, List, Optional
2
3 import torch
4 from torch.nn import Module
@@ -468,15 +468,18 @@ class Fp8MoEMethod(FusedMoEMethodBase):
468 requires_grad=False)
469 return
470
471 - def apply(self,
472 - layer: torch.nn.Module,
473 - x: torch.Tensor,
474 - router_logits: torch.Tensor,
475 - top_k: int,
476 - renormalize: bool,
477 - use_grouped_topk: bool,
478 - topk_group: Optional[int] = None,
479 - num_expert_group: Optional[int] = None) -> torch.Tensor:
471 + def apply(
472 + self,
473 + layer: torch.nn.Module,
474 + x: torch.Tensor,
475 + router_logits: torch.Tensor,
476 + top_k: int,
477 + renormalize: bool,
478 + use_grouped_topk: bool,
479 + topk_group: Optional[int] = None,
480 + num_expert_group: Optional[int] = None,
481 + custom_routing_function: Optional[Callable] = None,
482 + ) -> torch.Tensor:
483
484 from vllm.model_executor.layers.fused_moe import fused_experts
485
@@ -487,7 +490,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
490 top_k=top_k,
491 renormalize=renormalize,
492 topk_group=topk_group,
490 - num_expert_group=num_expert_group)
493 + num_expert_group=num_expert_group,
494 + custom_routing_function=custom_routing_function)
495
496 return fused_experts(x,
497 layer.w13_weight,
vllm/model_executor/layers/rotary_embedding.py
+15 -11
@@ -503,8 +503,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
503 dtype: torch.dtype,
504 short_factor: List[float],
505 long_factor: List[float],
506 - short_mscale: float = 1.0,
507 - long_mscale: float = 1.0,
506 + short_mscale: Optional[float] = None,
507 + long_mscale: Optional[float] = None,
508 ):
509 super().__init__()
510
@@ -523,18 +523,22 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
523 self.base = base
524 self.short_factor = short_factor
525 self.long_factor = long_factor
526 - self.short_mscale = short_mscale
527 - self.long_mscale = long_mscale
528 -
529 - scale = (self.max_position_embeddings /
530 - self.original_max_position_embeddings)
526
527 + scale = self.max_position_embeddings / \
528 + self.original_max_position_embeddings
529 if scale <= 1.0:
533 - self.scaling_factor = 1.0
530 + scaling_factor = 1.0
531 else:
535 - self.scaling_factor = math.sqrt(
532 + scaling_factor = math.sqrt(
533 1 + math.log(scale) /
534 math.log(self.original_max_position_embeddings))
535 + if short_mscale is None:
536 + short_mscale = scaling_factor
537 + if long_mscale is None:
538 + long_mscale = scaling_factor
539 +
540 + self.short_mscale = short_mscale
541 + self.long_mscale = long_mscale
542
543 short_cache = self._compute_cos_sin_cache(
544 original_max_position_embeddings, short_factor, short_mscale)
@@ -571,8 +575,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
575 inv_freq = self._compute_inv_freq(rescale_factors)
576 t = torch.arange(max_position_embeddings, dtype=torch.float)
577 freqs = torch.einsum("i,j -> ij", t, inv_freq)
574 - cos = freqs.cos() * mscale * self.scaling_factor
575 - sin = freqs.sin() * mscale * self.scaling_factor
578 + cos = freqs.cos() * mscale
579 + sin = freqs.sin() * mscale
580 cache = torch.cat((cos, sin), dim=-1)
581 return cache
582
vllm/model_executor/models/__init__.py
+1
@@ -50,6 +50,7 @@ _GENERATION_MODELS = {
50 "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
51 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
52 "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
53 + "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
54 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
55 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
56 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
vllm/model_executor/models/phimoe.py new
+620
@@ -0,0 +1,620 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only PhiMoE model."""
24 +from typing import Iterable, List, Optional, Tuple
25 +
26 +import torch
27 +from torch import nn
28 +from transformers.configuration_utils import PretrainedConfig
29 +
30 +from vllm.attention import Attention, AttentionMetadata
31 +from vllm.config import CacheConfig, LoRAConfig
32 +from vllm.distributed import get_tensor_model_parallel_world_size
33 +from vllm.model_executor.layers.fused_moe import FusedMoE
34 +from vllm.model_executor.layers.linear import (QKVParallelLinear,
35 + ReplicatedLinear,
36 + RowParallelLinear)
37 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 +from vllm.model_executor.layers.quantization.base_config import (
39 + QuantizationConfig)
40 +from vllm.model_executor.layers.rotary_embedding import get_rope
41 +from vllm.model_executor.layers.sampler import Sampler
42 +from vllm.model_executor.layers.vocab_parallel_embedding import (
43 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
44 +from vllm.model_executor.model_loader.weight_utils import (
45 + default_weight_loader, maybe_remap_kv_scale_name)
46 +from vllm.model_executor.sampling_metadata import SamplingMetadata
47 +from vllm.sequence import IntermediateTensors, SamplerOutput
48 +
49 +from .interfaces import SupportsLoRA
50 +
51 +
52 +class PhiMoEConfig(PretrainedConfig):
53 +
54 + model_type = "phimoe"
55 + keys_to_ignore_at_inference = ["past_key_values"]
56 +
57 + def __init__(
58 + self,
59 + vocab_size=32000,
60 + hidden_size=4096,
61 + intermediate_size=14336,
62 + num_hidden_layers=32,
63 + num_attention_heads=32,
64 + num_key_value_heads=8,
65 + hidden_act="silu",
66 + max_position_embeddings=4096 * 32,
67 + initializer_range=0.02,
68 + rms_norm_eps=1e-5,
69 + use_cache=True,
70 + pad_token_id=None,
71 + bos_token_id=1,
72 + eos_token_id=2,
73 + tie_word_embeddings=False,
74 + rope_theta=1e6,
75 + sliding_window=None,
76 + attention_dropout=0.0,
77 + num_experts_per_tok=2,
78 + num_local_experts=16,
79 + output_router_logits=False,
80 + router_aux_loss_coef=0.001,
81 + router_jitter_noise=0.0,
82 + attention_bias=False,
83 + lm_head_bias=False,
84 + **kwargs,
85 + ):
86 + self.vocab_size = vocab_size
87 + self.max_position_embeddings = max_position_embeddings
88 + self.hidden_size = hidden_size
89 + self.intermediate_size = intermediate_size
90 + self.num_hidden_layers = num_hidden_layers
91 + self.num_attention_heads = num_attention_heads
92 + self.sliding_window = sliding_window
93 + self.attention_bias = attention_bias
94 + self.lm_head_bias = lm_head_bias
95 + # for backward compatibility
96 + if num_key_value_heads is None:
97 + num_key_value_heads = num_attention_heads
98 +
99 + self.num_key_value_heads = num_key_value_heads
100 + self.hidden_act = hidden_act
101 + self.initializer_range = initializer_range
102 + self.rms_norm_eps = rms_norm_eps
103 + self.use_cache = use_cache
104 + self.rope_theta = rope_theta
105 + self.attention_dropout = attention_dropout
106 +
107 + self.num_experts_per_tok = num_experts_per_tok
108 + self.num_local_experts = num_local_experts
109 + self.output_router_logits = output_router_logits
110 + self.router_aux_loss_coef = router_aux_loss_coef
111 + self.router_jitter_noise = router_jitter_noise
112 + super().__init__(
113 + pad_token_id=pad_token_id,
114 + bos_token_id=bos_token_id,
115 + eos_token_id=eos_token_id,
116 + tie_word_embeddings=tie_word_embeddings,
117 + **kwargs,
118 + )
119 +
120 +
121 +class mp(torch.autograd.Function):
122 +
123 + @staticmethod
124 + def forward(
125 + ctx,
126 + scores: torch.Tensor,
127 + multiplier: torch.Tensor,
128 + selected_experts: torch.Tensor,
129 + masked_gates: torch.Tensor,
130 + mask_for_one: torch.Tensor,
131 + ):
132 + ctx.save_for_backward(multiplier, selected_experts, masked_gates)
133 + return multiplier * mask_for_one
134 +
135 + @staticmethod
136 + def backward(
137 + ctx,
138 + grad_at_output: torch.Tensor,
139 + ):
140 + multiplier, selected_experts, masked_gates = ctx.saved_tensors
141 +
142 + grad_at_output = grad_at_output * multiplier
143 +
144 + grad_at_scores_expaned = masked_gates * grad_at_output.mul(-1)
145 + grad_at_scores_expaned.scatter_add_(
146 + dim=-1,
147 + index=selected_experts,
148 + src=grad_at_output,
149 + )
150 +
151 + return (
152 + grad_at_scores_expaned,
153 + None,
154 + None,
155 + None,
156 + None,
157 + )
158 +
159 +
160 +def sparsemixer(scores, jitter_eps=0.01):
161 + ################ first expert ################
162 +
163 + with torch.no_grad():
164 + # compute mask for sparsity
165 + mask_logits_threshold, max_ind = scores.max(dim=-1, keepdim=True)
166 + factor = scores.abs().clamp(min=mask_logits_threshold)
167 + mask_logits_threshold = (
168 + (mask_logits_threshold - scores) / factor) > (2 * jitter_eps)
169 +
170 + # apply mask
171 + masked_gates = scores.masked_fill(mask_logits_threshold, float("-inf"))
172 + selected_experts = max_ind
173 +
174 + # compute scores for gradients
175 + masked_gates = torch.softmax(masked_gates, dim=-1)
176 + multiplier_o = masked_gates.gather(dim=-1, index=selected_experts)
177 +
178 + multiplier = multiplier_o
179 +
180 + # masked out first expert
181 + masked_scores = torch.scatter(
182 + scores,
183 + -1,
184 + selected_experts,
185 + float("-inf"),
186 + )
187 + with torch.no_grad():
188 + # compute mask for sparsity
189 + mask_logits_threshold, max_ind = masked_scores.max(dim=-1,
190 + keepdim=True)
191 + factor = scores.abs().clamp(min=mask_logits_threshold)
192 + mask_logits_threshold = (
193 + (mask_logits_threshold - scores) / factor) > (2 * jitter_eps)
194 +
195 + # apply mask
196 + masked_gates_top2 = masked_scores.masked_fill(mask_logits_threshold,
197 + float("-inf"))
198 + selected_experts_top2 = max_ind
199 + # compute scores for gradients
200 + masked_gates_top2 = torch.softmax(masked_gates_top2, dim=-1)
201 + multiplier_top2 = masked_gates_top2.gather(dim=-1,
202 + index=selected_experts_top2)
203 +
204 + multiplier = torch.concat((multiplier, multiplier_top2), dim=-1)
205 + selected_experts = torch.concat((selected_experts, selected_experts_top2),
206 + dim=-1)
207 +
208 + return (
209 + multiplier,
210 + selected_experts,
211 + )
212 +
213 +
214 +def phimoe_routing_function(
215 + hidden_states: torch.Tensor,
216 + gating_output: torch.Tensor,
217 + topk: int,
218 + renormalize: bool,
219 +):
220 + assert hidden_states.shape[0] == gating_output.shape[0], (
221 + "Number of tokens mismatch")
222 + assert topk == 2, "Only top-2 routing is supported"
223 + assert renormalize is False, "Renormalization is not supported"
224 +
225 + topk_weights, topk_ids = sparsemixer(gating_output)
226 + return topk_weights, topk_ids
227 +
228 +
229 +class PhiMoE(nn.Module):
230 + """A tensor-parallel MoE implementation for PhiMoE that shards each expert
231 + across all ranks.
232 +
233 + Each expert's weights are sharded across all ranks and a fused MoE
234 + kernel is used for the forward pass, and finally we reduce the outputs
235 + across ranks.
236 + """
237 +
238 + def __init__(
239 + self,
240 + num_experts: int,
241 + top_k: int,
242 + hidden_size: int,
243 + intermediate_size: int,
244 + params_dtype: Optional[torch.dtype] = None,
245 + quant_config: Optional[QuantizationConfig] = None,
246 + tp_size: Optional[int] = None,
247 + ):
248 + super().__init__()
249 + self.hidden_size = hidden_size
250 +
251 + # Gate always runs at half / full precision for now.
252 + self.gate = ReplicatedLinear(
253 + hidden_size,
254 + num_experts,
255 + bias=False,
256 + params_dtype=params_dtype,
257 + quant_config=None,
258 + )
259 +
260 + self.experts = FusedMoE(
261 + num_experts=num_experts,
262 + top_k=top_k,
263 + hidden_size=hidden_size,
264 + intermediate_size=intermediate_size,
265 + params_dtype=params_dtype,
266 + reduce_results=True,
267 + renormalize=False,
268 + quant_config=quant_config,
269 + tp_size=tp_size,
270 + custom_routing_function=phimoe_routing_function)
271 +
272 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
273 + # NOTE: hidden_states can have either 1D or 2D shape.
274 + orig_shape = hidden_states.shape
275 + hidden_states = hidden_states.view(-1, self.hidden_size)
276 + # router_logits: (num_tokens, n_experts)
277 + router_logits, _ = self.gate(hidden_states)
278 + final_hidden_states = self.experts(hidden_states, router_logits)
279 + return final_hidden_states.view(orig_shape)
280 +
281 +
282 +class PhiMoEAttention(nn.Module):
283 +
284 + def __init__(
285 + self,
286 + hidden_size: int,
287 + num_heads: int,
288 + num_kv_heads: int,
289 + max_position: int = 4096 * 32,
290 + rope_theta: float = 10000,
291 + cache_config: Optional[CacheConfig] = None,
292 + quant_config: Optional[QuantizationConfig] = None,
293 + rope_scaling: Optional[dict] = None,
294 + ) -> None:
295 + super().__init__()
296 + self.hidden_size = hidden_size
297 + tp_size = get_tensor_model_parallel_world_size()
298 + self.total_num_heads = num_heads
299 + assert self.total_num_heads % tp_size == 0
300 + self.num_heads = self.total_num_heads // tp_size
301 + self.total_num_kv_heads = num_kv_heads
302 + if self.total_num_kv_heads >= tp_size:
303 + # Number of KV heads is greater than TP size, so we partition
304 + # the KV heads across multiple tensor parallel GPUs.
305 + assert self.total_num_kv_heads % tp_size == 0
306 + else:
307 + # Number of KV heads is less than TP size, so we replicate
308 + # the KV heads across multiple tensor parallel GPUs.
309 + assert tp_size % self.total_num_kv_heads == 0
310 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
311 + self.head_dim = hidden_size // self.total_num_heads
312 + self.q_size = self.num_heads * self.head_dim
313 + self.kv_size = self.num_kv_heads * self.head_dim
314 + self.scaling = self.head_dim**-0.5
315 + self.rope_theta = rope_theta
316 + self.rope_scaling = rope_scaling
317 +
318 + self.qkv_proj = QKVParallelLinear(
319 + hidden_size,
320 + self.head_dim,
321 + self.total_num_heads,
322 + self.total_num_kv_heads,
323 + bias=True,
324 + quant_config=None,
325 + )
326 + self.o_proj = RowParallelLinear(
327 + self.total_num_heads * self.head_dim,
328 + hidden_size,
329 + bias=True,
330 + quant_config=None,
331 + )
332 + self.rotary_emb = get_rope(
333 + self.head_dim,
334 + rotary_dim=self.head_dim,
335 + max_position=max_position,
336 + base=int(self.rope_theta),
337 + is_neox_style=True,
338 + rope_scaling=self.rope_scaling,
339 + )
340 + self.attn = Attention(
341 + self.num_heads,
342 + self.head_dim,
343 + self.scaling,
344 + num_kv_heads=self.num_kv_heads,
345 + cache_config=cache_config,
346 + quant_config=quant_config,
347 + )
348 +
349 + def forward(
350 + self,
351 + positions: torch.Tensor,
352 + hidden_states: torch.Tensor,
353 + kv_cache: torch.Tensor,
354 + attn_metadata: AttentionMetadata,
355 + ) -> torch.Tensor:
356 + qkv, _ = self.qkv_proj(hidden_states)
357 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
358 + q, k = self.rotary_emb(positions, q, k)
359 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
360 + output, _ = self.o_proj(attn_output)
361 + return output
362 +
363 +
364 +class PhiMoEDecoderLayer(nn.Module):
365 +
366 + def __init__(
367 + self,
368 + config: PhiMoEConfig,
369 + cache_config: Optional[CacheConfig] = None,
370 + quant_config: Optional[QuantizationConfig] = None,
371 + ) -> None:
372 + super().__init__()
373 + self.hidden_size = config.hidden_size
374 + # Requires transformers > 4.32.0
375 + rope_theta = getattr(config, "rope_theta", 10000)
376 + self.self_attn = PhiMoEAttention(
377 + hidden_size=self.hidden_size,
378 + num_heads=config.num_attention_heads,
379 + max_position=config.max_position_embeddings,
380 + num_kv_heads=config.num_key_value_heads,
381 + rope_theta=rope_theta,
382 + cache_config=cache_config,
383 + quant_config=quant_config,
384 + rope_scaling=config.rope_scaling,
385 + )
386 + self.block_sparse_moe = PhiMoE(
387 + num_experts=config.num_local_experts,
388 + top_k=config.num_experts_per_tok,
389 + hidden_size=config.hidden_size,
390 + intermediate_size=config.intermediate_size,
391 + quant_config=quant_config,
392 + )
393 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
394 + eps=config.rms_norm_eps,
395 + elementwise_affine=True)
396 + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
397 + eps=config.rms_norm_eps,
398 + elementwise_affine=True)
399 +
400 + def forward(
401 + self,
402 + positions: torch.Tensor,
403 + hidden_states: torch.Tensor,
404 + kv_cache: torch.Tensor,
405 + attn_metadata: AttentionMetadata,
406 + residual: Optional[torch.Tensor],
407 + ) -> torch.Tensor:
408 + residual = hidden_states
409 +
410 + # Self Attention
411 + hidden_states = self.input_layernorm(hidden_states)
412 +
413 + hidden_states = self.self_attn(
414 + positions=positions,
415 + hidden_states=hidden_states,
416 + kv_cache=kv_cache,
417 + attn_metadata=attn_metadata,
418 + )
419 + hidden_states = hidden_states + residual
420 +
421 + # Fully Connected
422 + residual = hidden_states
423 + hidden_states = self.post_attention_layernorm(hidden_states)
424 + hidden_states = self.block_sparse_moe(hidden_states)
425 +
426 + hidden_states = hidden_states + residual
427 + return hidden_states, residual
428 +
429 +
430 +class PhiMoEModel(nn.Module):
431 +
432 + def __init__(
433 + self,
434 + config: PhiMoEConfig,
435 + cache_config: Optional[CacheConfig] = None,
436 + quant_config: Optional[QuantizationConfig] = None,
437 + lora_config: Optional[LoRAConfig] = None,
438 + ) -> None:
439 + super().__init__()
440 + self.padding_idx = config.pad_token_id
441 + lora_vocab = ((lora_config.lora_extra_vocab_size *
442 + (lora_config.max_loras or 1)) if lora_config else 0)
443 + self.vocab_size = config.vocab_size + lora_vocab
444 + self.org_vocab_size = config.vocab_size
445 +
446 + self.embed_tokens = VocabParallelEmbedding(
447 + self.vocab_size,
448 + config.hidden_size,
449 + org_num_embeddings=config.vocab_size,
450 + )
451 + self.layers = nn.ModuleList([
452 + PhiMoEDecoderLayer(config, cache_config, quant_config=quant_config)
453 + for _ in range(config.num_hidden_layers)
454 + ])
455 + self.norm = nn.LayerNorm(config.hidden_size,
456 + eps=config.rms_norm_eps,
457 + elementwise_affine=True)
458 +
459 + def forward(
460 + self,
461 + input_ids: torch.Tensor,
462 + positions: torch.Tensor,
463 + kv_caches: List[torch.Tensor],
464 + attn_metadata: AttentionMetadata,
465 + ) -> torch.Tensor:
466 + hidden_states = self.embed_tokens(input_ids)
467 + residual = None
468 + for i in range(len(self.layers)):
469 + layer = self.layers[i]
470 + hidden_states, residual = layer(positions, hidden_states,
471 + kv_caches[i], attn_metadata,
472 + residual)
473 + hidden_states = self.norm(hidden_states)
474 + return hidden_states
475 +
476 +
477 +class PhiMoEForCausalLM(nn.Module, SupportsLoRA):
478 + fall_back_to_pt_during_load = False
479 +
480 + packed_modules_mapping = {
481 + "qkv_proj": [
482 + "q_proj",
483 + "k_proj",
484 + "v_proj",
485 + ],
486 + }
487 +
488 + # LoRA specific attributes
489 + supported_lora_modules = [
490 + "qkv_proj",
491 + "o_proj",
492 + "embed_tokens",
493 + "lm_head",
494 + ]
495 + embedding_modules = {
496 + "embed_tokens": "input_embeddings",
497 + "lm_head": "output_embeddings",
498 + }
499 + embedding_padding_modules = ["lm_head"]
500 +
501 + def __init__(
502 + self,
503 + config: PhiMoEConfig,
504 + cache_config: Optional[CacheConfig] = None,
505 + quant_config: Optional[QuantizationConfig] = None,
506 + lora_config: Optional[LoRAConfig] = None,
507 + ) -> None:
508 + super().__init__()
509 +
510 + self.config = config
511 + self.lora_config = lora_config
512 +
513 + self.model = PhiMoEModel(config,
514 + cache_config,
515 + quant_config,
516 + lora_config=lora_config)
517 + self.unpadded_vocab_size = config.vocab_size
518 + if lora_config:
519 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
520 + self.lm_head = ParallelLMHead(
521 + self.unpadded_vocab_size,
522 + config.hidden_size,
523 + org_num_embeddings=config.vocab_size,
524 + padding_size=(
525 + DEFAULT_VOCAB_PADDING_SIZE
526 + # We need bigger padding if using lora for kernel
527 + # compatibility
528 + if not lora_config else lora_config.lora_vocab_padding_size),
529 + quant_config=None,
530 + bias=True,
531 + )
532 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
533 + config.vocab_size)
534 + self.sampler = Sampler()
535 +
536 + def forward(
537 + self,
538 + input_ids: torch.Tensor,
539 + positions: torch.Tensor,
540 + kv_caches: List[torch.Tensor],
541 + attn_metadata: AttentionMetadata,
542 + intermediate_tensors: Optional[IntermediateTensors] = None,
543 + ) -> torch.Tensor:
544 + hidden_states = self.model(input_ids, positions, kv_caches,
545 + attn_metadata)
546 + return hidden_states
547 +
548 + def compute_logits(self, hidden_states: torch.Tensor,
549 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
550 + logits = self.logits_processor(self.lm_head, hidden_states,
551 + sampling_metadata)
552 + return logits
553 +
554 + def sample(
555 + self,
556 + logits: Optional[torch.Tensor],
557 + sampling_metadata: SamplingMetadata,
558 + ) -> Optional[SamplerOutput]:
559 + next_tokens = self.sampler(logits, sampling_metadata)
560 + return next_tokens
561 +
562 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
563 + stacked_params_mapping = [
564 + # (param_name, shard_name, shard_id)
565 + ("qkv_proj", "q_proj", "q"),
566 + ("qkv_proj", "k_proj", "k"),
567 + ("qkv_proj", "v_proj", "v"),
568 + ]
569 +
570 + expert_params_mapping = FusedMoE.make_expert_params_mapping(
571 + ckpt_gate_proj_name="w1",
572 + ckpt_down_proj_name="w2",
573 + ckpt_up_proj_name="w3",
574 + num_experts=self.config.num_local_experts)
575 +
576 + params_dict = dict(self.named_parameters())
577 + for name, loaded_weight in weights:
578 + if "rotary_emb.inv_freq" in name:
579 + continue
580 +
581 + for param_name, weight_name, shard_id in stacked_params_mapping:
582 + if weight_name not in name:
583 + continue
584 + name = name.replace(weight_name, param_name)
585 + # Skip loading extra bias for GPTQ models.
586 + if name.endswith(".bias") and name not in params_dict:
587 + continue
588 + param = params_dict[name]
589 + weight_loader = param.weight_loader
590 + weight_loader(param, loaded_weight, shard_id)
591 + break
592 + else:
593 + for mapping in expert_params_mapping:
594 + param_name, weight_name, expert_id, shard_id = mapping
595 + if weight_name not in name:
596 + continue
597 + name = name.replace(weight_name, param_name)
598 + param = params_dict[name]
599 + weight_loader = param.weight_loader
600 + weight_loader(
601 + param,
602 + loaded_weight,
603 + weight_name,
604 + shard_id=shard_id,
605 + expert_id=expert_id,
606 + )
607 + break
608 + else:
609 + # Skip loading extra bias for GPTQ models.
610 + if name.endswith(".bias") and name not in params_dict:
611 + continue
612 + # Remapping the name of FP8 kv-scale.
613 + name = maybe_remap_kv_scale_name(name, params_dict)
614 + if name is None:
615 + continue
616 +
617 + param = params_dict[name]
618 + weight_loader = getattr(param, "weight_loader",
619 + default_weight_loader)
620 + weight_loader(param, loaded_weight)