A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model] Adding support for MSFT Phi-3.5-MoE (#7729)
Co-authored-by: Your Name <you@example.com> Co-authored-by: Zeqi Lin <zelin@microsoft.com> Co-authored-by: Zeqi Lin <Zeqi.Lin@microsoft.com>
Wenxiang committed
Aug 31, 2024 at 03:42 UTC
1248e8506a4d98b4f15cbfe729cf2af42fb4223a
13 files changed
+1255
-82
docs/source/models/supported_models.rst
+4
@@ -147,6 +147,10 @@ Decoder-only Language Models
147
- Phi-3-Small
148
- :code:`microsoft/Phi-3-small-8k-instruct`, :code:`microsoft/Phi-3-small-128k-instruct`, etc.
149
-
150
+ * - :code:`PhiMoEForCausalLM`
151
+ - Phi-3.5-MoE
152
+ - :code:`microsoft/Phi-3.5-MoE-instruct`, etc.
153
+ -
154
* - :code:`PersimmonForCausalLM`
155
- Persimmon
156
- :code:`adept/persimmon-8b-base`, :code:`adept/persimmon-8b-chat`, etc.
tests/models/test_phimoe.py
new
+111
@@ -0,0 +1,111 @@
1
+"""Compare the outputs of HF and vLLM for moe models using greedy sampling.
2
+
3
+Run `pytest tests/models/test_phimoe.py`.
4
+"""
5
+import pytest
6
+import torch
7
+
8
+from vllm.utils import is_cpu
9
+
10
+from .utils import check_logprobs_close
11
+
12
+MODELS = [
13
+ "microsoft/Phi-3.5-MoE-instruct",
14
+]
15
+
16
+
17
+def test_phimoe_routing_function():
18
+ from vllm.model_executor.models.phimoe import phimoe_routing_function
19
+ test_case = {
20
+ 0: {
21
+ "hidden_states":
22
+ torch.tensor([1, 2, 3, 4, 5, 6, 7, 8],
23
+ dtype=torch.float32,
24
+ requires_grad=False).view(4, 2),
25
+ "gating_output":
26
+ torch.tensor([0.1, 0.2, 0.3, 0.4],
27
+ dtype=torch.float32,
28
+ requires_grad=False),
29
+ "topk":
30
+ 2,
31
+ "renormalize":
32
+ False,
33
+ },
34
+ 1: {
35
+ "hidden_states":
36
+ torch.tensor([1, 2, 3, 4, 5, 6, 7, 8],
37
+ dtype=torch.float32,
38
+ requires_grad=False).view(4, 2),
39
+ "gating_output":
40
+ torch.tensor([0.4, 0.2, 0.3, 0.4],
41
+ dtype=torch.float32,
42
+ requires_grad=False),
43
+ "topk":
44
+ 2,
45
+ "renormalize":
46
+ False,
47
+ }
48
+ }
49
+
50
+ ground_truth = {
51
+ 0: {
52
+ "topk_weights":
53
+ torch.tensor([1., 1.], dtype=torch.float32, requires_grad=False),
54
+ "topk_ids":
55
+ torch.tensor([3, 2], dtype=torch.long, requires_grad=False),
56
+ },
57
+ 1: {
58
+ "topk_weights":
59
+ torch.tensor([0.5, 1.], dtype=torch.float32, requires_grad=False),
60
+ "topk_ids":
61
+ torch.tensor([0, 3], dtype=torch.long, requires_grad=False),
62
+ }
63
+ }
64
+
65
+ for test_id in test_case:
66
+ topk_weights, topk_ids = phimoe_routing_function(**test_case[test_id])
67
+ assert torch.allclose(topk_weights,
68
+ ground_truth[test_id]["topk_weights"])
69
+ assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
70
+
71
+
72
+def get_gpu_memory():
73
+ try:
74
+ props = torch.cuda.get_device_properties(torch.cuda.current_device())
75
+ gpu_memory = props.total_memory / (1024**3)
76
+ return gpu_memory
77
+ except Exception:
78
+ return 0
79
+
80
+
81
+@pytest.mark.skipif(condition=is_cpu(),
82
+ reason="This test takes a lot time to run on CPU, "
83
+ "and vllm CI's disk space is not enough for this model.")
84
+@pytest.mark.skipif(condition=get_gpu_memory() < 100,
85
+ reason="Skip this test if GPU memory is insufficient.")
86
+@pytest.mark.parametrize("model", MODELS)
87
+@pytest.mark.parametrize("dtype", ["bfloat16"])
88
+@pytest.mark.parametrize("max_tokens", [64])
89
+@pytest.mark.parametrize("num_logprobs", [5])
90
+def test_models(
91
+ hf_runner,
92
+ vllm_runner,
93
+ example_prompts,
94
+ model: str,
95
+ dtype: str,
96
+ max_tokens: int,
97
+ num_logprobs: int,
98
+) -> None:
99
+ with hf_runner(model, dtype=dtype) as hf_model:
100
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
101
+ example_prompts, max_tokens, num_logprobs)
102
+
103
+ with vllm_runner(model, dtype=dtype) as vllm_model:
104
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
105
+ example_prompts, max_tokens, num_logprobs)
106
+ check_logprobs_close(
107
+ outputs_0_lst=hf_outputs,
108
+ outputs_1_lst=vllm_outputs,
109
+ name_0="hf",
110
+ name_1="vllm",
111
+ )
vllm/model_executor/layers/fused_moe/configs/E=16,N=3200,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json
new
+130
@@ -0,0 +1,130 @@
1
+{
2
+ "3328": {
3
+ "BLOCK_SIZE_M": 64,
4
+ "BLOCK_SIZE_N": 256,
5
+ "BLOCK_SIZE_K": 64,
6
+ "GROUP_SIZE_M": 16,
7
+ "num_warps": 4,
8
+ "num_stages": 2
9
+ },
10
+ "1024": {
11
+ "BLOCK_SIZE_M": 64,
12
+ "BLOCK_SIZE_N": 256,
13
+ "BLOCK_SIZE_K": 32,
14
+ "GROUP_SIZE_M": 32,
15
+ "num_warps": 4,
16
+ "num_stages": 4
17
+ },
18
+ "3072": {
19
+ "BLOCK_SIZE_M": 64,
20
+ "BLOCK_SIZE_N": 256,
21
+ "BLOCK_SIZE_K": 64,
22
+ "GROUP_SIZE_M": 32,
23
+ "num_warps": 4,
24
+ "num_stages": 2
25
+ },
26
+ "256": {
27
+ "BLOCK_SIZE_M": 32,
28
+ "BLOCK_SIZE_N": 256,
29
+ "BLOCK_SIZE_K": 128,
30
+ "GROUP_SIZE_M": 8,
31
+ "num_warps": 4,
32
+ "num_stages": 4
33
+ },
34
+ "768": {
35
+ "BLOCK_SIZE_M": 128,
36
+ "BLOCK_SIZE_N": 128,
37
+ "BLOCK_SIZE_K": 64,
38
+ "GROUP_SIZE_M": 8,
39
+ "num_warps": 4,
40
+ "num_stages": 4
41
+ },
42
+ "1792": {
43
+ "BLOCK_SIZE_M": 128,
44
+ "BLOCK_SIZE_N": 128,
45
+ "BLOCK_SIZE_K": 64,
46
+ "GROUP_SIZE_M": 16,
47
+ "num_warps": 4,
48
+ "num_stages": 4
49
+ },
50
+ "2560": {
51
+ "BLOCK_SIZE_M": 64,
52
+ "BLOCK_SIZE_N": 256,
53
+ "BLOCK_SIZE_K": 64,
54
+ "GROUP_SIZE_M": 32,
55
+ "num_warps": 4,
56
+ "num_stages": 2
57
+ },
58
+ "2816": {
59
+ "BLOCK_SIZE_M": 128,
60
+ "BLOCK_SIZE_N": 128,
61
+ "BLOCK_SIZE_K": 64,
62
+ "GROUP_SIZE_M": 16,
63
+ "num_warps": 4,
64
+ "num_stages": 4
65
+ },
66
+ "3584": {
67
+ "BLOCK_SIZE_M": 64,
68
+ "BLOCK_SIZE_N": 256,
69
+ "BLOCK_SIZE_K": 64,
70
+ "GROUP_SIZE_M": 32,
71
+ "num_warps": 4,
72
+ "num_stages": 2
73
+ },
74
+ "1536": {
75
+ "BLOCK_SIZE_M": 64,
76
+ "BLOCK_SIZE_N": 256,
77
+ "BLOCK_SIZE_K": 64,
78
+ "GROUP_SIZE_M": 64,
79
+ "num_warps": 4,
80
+ "num_stages": 2
81
+ },
82
+ "2048": {
83
+ "BLOCK_SIZE_M": 64,
84
+ "BLOCK_SIZE_N": 256,
85
+ "BLOCK_SIZE_K": 64,
86
+ "GROUP_SIZE_M": 64,
87
+ "num_warps": 4,
88
+ "num_stages": 2
89
+ },
90
+ "512": {
91
+ "BLOCK_SIZE_M": 64,
92
+ "BLOCK_SIZE_N": 256,
93
+ "BLOCK_SIZE_K": 64,
94
+ "GROUP_SIZE_M": 8,
95
+ "num_warps": 4,
96
+ "num_stages": 4
97
+ },
98
+ "3840": {
99
+ "BLOCK_SIZE_M": 128,
100
+ "BLOCK_SIZE_N": 128,
101
+ "BLOCK_SIZE_K": 64,
102
+ "GROUP_SIZE_M": 16,
103
+ "num_warps": 4,
104
+ "num_stages": 4
105
+ },
106
+ "1280": {
107
+ "BLOCK_SIZE_M": 64,
108
+ "BLOCK_SIZE_N": 256,
109
+ "BLOCK_SIZE_K": 64,
110
+ "GROUP_SIZE_M": 64,
111
+ "num_warps": 4,
112
+ "num_stages": 2
113
+ },
114
+ "2304": {
115
+ "BLOCK_SIZE_M": 64,
116
+ "BLOCK_SIZE_N": 256,
117
+ "BLOCK_SIZE_K": 64,
118
+ "GROUP_SIZE_M": 32,
119
+ "num_warps": 4,
120
+ "num_stages": 2
121
+ },
122
+ "4096": {
123
+ "BLOCK_SIZE_M": 64,
124
+ "BLOCK_SIZE_N": 256,
125
+ "BLOCK_SIZE_K": 64,
126
+ "GROUP_SIZE_M": 32,
127
+ "num_warps": 4,
128
+ "num_stages": 2
129
+ }
130
+}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=16,N=6400,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json
new
+130
@@ -0,0 +1,130 @@
1
+{
2
+ "3840": {
3
+ "BLOCK_SIZE_M": 128,
4
+ "BLOCK_SIZE_N": 128,
5
+ "BLOCK_SIZE_K": 64,
6
+ "GROUP_SIZE_M": 8,
7
+ "num_warps": 4,
8
+ "num_stages": 4
9
+ },
10
+ "1792": {
11
+ "BLOCK_SIZE_M": 128,
12
+ "BLOCK_SIZE_N": 128,
13
+ "BLOCK_SIZE_K": 64,
14
+ "GROUP_SIZE_M": 8,
15
+ "num_warps": 4,
16
+ "num_stages": 4
17
+ },
18
+ "3584": {
19
+ "BLOCK_SIZE_M": 64,
20
+ "BLOCK_SIZE_N": 256,
21
+ "BLOCK_SIZE_K": 64,
22
+ "GROUP_SIZE_M": 16,
23
+ "num_warps": 4,
24
+ "num_stages": 2
25
+ },
26
+ "512": {
27
+ "BLOCK_SIZE_M": 64,
28
+ "BLOCK_SIZE_N": 256,
29
+ "BLOCK_SIZE_K": 64,
30
+ "GROUP_SIZE_M": 16,
31
+ "num_warps": 4,
32
+ "num_stages": 2
33
+ },
34
+ "3072": {
35
+ "BLOCK_SIZE_M": 64,
36
+ "BLOCK_SIZE_N": 256,
37
+ "BLOCK_SIZE_K": 64,
38
+ "GROUP_SIZE_M": 32,
39
+ "num_warps": 4,
40
+ "num_stages": 2
41
+ },
42
+ "2048": {
43
+ "BLOCK_SIZE_M": 64,
44
+ "BLOCK_SIZE_N": 256,
45
+ "BLOCK_SIZE_K": 64,
46
+ "GROUP_SIZE_M": 16,
47
+ "num_warps": 4,
48
+ "num_stages": 2
49
+ },
50
+ "2816": {
51
+ "BLOCK_SIZE_M": 128,
52
+ "BLOCK_SIZE_N": 256,
53
+ "BLOCK_SIZE_K": 32,
54
+ "GROUP_SIZE_M": 32,
55
+ "num_warps": 8,
56
+ "num_stages": 4
57
+ },
58
+ "1280": {
59
+ "BLOCK_SIZE_M": 64,
60
+ "BLOCK_SIZE_N": 256,
61
+ "BLOCK_SIZE_K": 64,
62
+ "GROUP_SIZE_M": 64,
63
+ "num_warps": 4,
64
+ "num_stages": 2
65
+ },
66
+ "768": {
67
+ "BLOCK_SIZE_M": 128,
68
+ "BLOCK_SIZE_N": 128,
69
+ "BLOCK_SIZE_K": 64,
70
+ "GROUP_SIZE_M": 1,
71
+ "num_warps": 4,
72
+ "num_stages": 4
73
+ },
74
+ "4096": {
75
+ "BLOCK_SIZE_M": 128,
76
+ "BLOCK_SIZE_N": 128,
77
+ "BLOCK_SIZE_K": 64,
78
+ "GROUP_SIZE_M": 8,
79
+ "num_warps": 4,
80
+ "num_stages": 4
81
+ },
82
+ "3328": {
83
+ "BLOCK_SIZE_M": 64,
84
+ "BLOCK_SIZE_N": 256,
85
+ "BLOCK_SIZE_K": 64,
86
+ "GROUP_SIZE_M": 32,
87
+ "num_warps": 4,
88
+ "num_stages": 2
89
+ },
90
+ "2560": {
91
+ "BLOCK_SIZE_M": 128,
92
+ "BLOCK_SIZE_N": 128,
93
+ "BLOCK_SIZE_K": 64,
94
+ "GROUP_SIZE_M": 8,
95
+ "num_warps": 4,
96
+ "num_stages": 4
97
+ },
98
+ "1024": {
99
+ "BLOCK_SIZE_M": 64,
100
+ "BLOCK_SIZE_N": 256,
101
+ "BLOCK_SIZE_K": 32,
102
+ "GROUP_SIZE_M": 8,
103
+ "num_warps": 4,
104
+ "num_stages": 4
105
+ },
106
+ "2304": {
107
+ "BLOCK_SIZE_M": 64,
108
+ "BLOCK_SIZE_N": 256,
109
+ "BLOCK_SIZE_K": 64,
110
+ "GROUP_SIZE_M": 16,
111
+ "num_warps": 4,
112
+ "num_stages": 2
113
+ },
114
+ "1536": {
115
+ "BLOCK_SIZE_M": 64,
116
+ "BLOCK_SIZE_N": 256,
117
+ "BLOCK_SIZE_K": 64,
118
+ "GROUP_SIZE_M": 32,
119
+ "num_warps": 4,
120
+ "num_stages": 2
121
+ },
122
+ "256": {
123
+ "BLOCK_SIZE_M": 64,
124
+ "BLOCK_SIZE_N": 256,
125
+ "BLOCK_SIZE_K": 64,
126
+ "GROUP_SIZE_M": 1,
127
+ "num_warps": 4,
128
+ "num_stages": 4
129
+ }
130
+}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=16,N=800,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json
new
+130
@@ -0,0 +1,130 @@
1
+{
2
+ "2048": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 256,
5
+ "BLOCK_SIZE_K": 32,
6
+ "GROUP_SIZE_M": 8,
7
+ "num_warps": 4,
8
+ "num_stages": 4
9
+ },
10
+ "1792": {
11
+ "BLOCK_SIZE_M": 128,
12
+ "BLOCK_SIZE_N": 256,
13
+ "BLOCK_SIZE_K": 64,
14
+ "GROUP_SIZE_M": 16,
15
+ "num_warps": 8,
16
+ "num_stages": 4
17
+ },
18
+ "512": {
19
+ "BLOCK_SIZE_M": 64,
20
+ "BLOCK_SIZE_N": 64,
21
+ "BLOCK_SIZE_K": 256,
22
+ "GROUP_SIZE_M": 8,
23
+ "num_warps": 4,
24
+ "num_stages": 4
25
+ },
26
+ "3328": {
27
+ "BLOCK_SIZE_M": 128,
28
+ "BLOCK_SIZE_N": 256,
29
+ "BLOCK_SIZE_K": 64,
30
+ "GROUP_SIZE_M": 64,
31
+ "num_warps": 8,
32
+ "num_stages": 2
33
+ },
34
+ "3072": {
35
+ "BLOCK_SIZE_M": 64,
36
+ "BLOCK_SIZE_N": 64,
37
+ "BLOCK_SIZE_K": 64,
38
+ "GROUP_SIZE_M": 1,
39
+ "num_warps": 4,
40
+ "num_stages": 2
41
+ },
42
+ "2560": {
43
+ "BLOCK_SIZE_M": 64,
44
+ "BLOCK_SIZE_N": 128,
45
+ "BLOCK_SIZE_K": 32,
46
+ "GROUP_SIZE_M": 8,
47
+ "num_warps": 4,
48
+ "num_stages": 4
49
+ },
50
+ "768": {
51
+ "BLOCK_SIZE_M": 64,
52
+ "BLOCK_SIZE_N": 64,
53
+ "BLOCK_SIZE_K": 256,
54
+ "GROUP_SIZE_M": 16,
55
+ "num_warps": 4,
56
+ "num_stages": 2
57
+ },
58
+ "2816": {
59
+ "BLOCK_SIZE_M": 16,
60
+ "BLOCK_SIZE_N": 256,
61
+ "BLOCK_SIZE_K": 64,
62
+ "GROUP_SIZE_M": 1,
63
+ "num_warps": 4,
64
+ "num_stages": 2
65
+ },
66
+ "256": {
67
+ "BLOCK_SIZE_M": 16,
68
+ "BLOCK_SIZE_N": 32,
69
+ "BLOCK_SIZE_K": 64,
70
+ "GROUP_SIZE_M": 16,
71
+ "num_warps": 4,
72
+ "num_stages": 4
73
+ },
74
+ "4096": {
75
+ "BLOCK_SIZE_M": 32,
76
+ "BLOCK_SIZE_N": 256,
77
+ "BLOCK_SIZE_K": 64,
78
+ "GROUP_SIZE_M": 64,
79
+ "num_warps": 4,
80
+ "num_stages": 4
81
+ },
82
+ "1024": {
83
+ "BLOCK_SIZE_M": 32,
84
+ "BLOCK_SIZE_N": 256,
85
+ "BLOCK_SIZE_K": 256,
86
+ "GROUP_SIZE_M": 1,
87
+ "num_warps": 4,
88
+ "num_stages": 2
89
+ },
90
+ "2304": {
91
+ "BLOCK_SIZE_M": 32,
92
+ "BLOCK_SIZE_N": 64,
93
+ "BLOCK_SIZE_K": 256,
94
+ "GROUP_SIZE_M": 8,
95
+ "num_warps": 8,
96
+ "num_stages": 2
97
+ },
98
+ "1280": {
99
+ "BLOCK_SIZE_M": 64,
100
+ "BLOCK_SIZE_N": 64,
101
+ "BLOCK_SIZE_K": 64,
102
+ "GROUP_SIZE_M": 16,
103
+ "num_warps": 4,
104
+ "num_stages": 4
105
+ },
106
+ "3840": {
107
+ "BLOCK_SIZE_M": 16,
108
+ "BLOCK_SIZE_N": 128,
109
+ "BLOCK_SIZE_K": 256,
110
+ "GROUP_SIZE_M": 1,
111
+ "num_warps": 4,
112
+ "num_stages": 4
113
+ },
114
+ "1536": {
115
+ "BLOCK_SIZE_M": 32,
116
+ "BLOCK_SIZE_N": 256,
117
+ "BLOCK_SIZE_K": 256,
118
+ "GROUP_SIZE_M": 1,
119
+ "num_warps": 4,
120
+ "num_stages": 2
121
+ },
122
+ "3584": {
123
+ "BLOCK_SIZE_M": 16,
124
+ "BLOCK_SIZE_N": 128,
125
+ "BLOCK_SIZE_K": 32,
126
+ "GROUP_SIZE_M": 1,
127
+ "num_warps": 8,
128
+ "num_stages": 4
129
+ }
130
+}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/fused_moe.py
+14
-5
@@ -2,7 +2,7 @@
2
import functools
3
import json
4
import os
5
-from typing import Any, Dict, Optional, Tuple
5
+from typing import Any, Callable, Dict, Optional, Tuple
6
7
import torch
8
import triton
@@ -446,7 +446,8 @@ def fused_marlin_moe(hidden_states: torch.Tensor,
446
rand_perm1: torch.Tensor,
447
rand_perm2: torch.Tensor,
448
topk: int,
449
- renormalize: bool,
449
+ custom_routing_function: Optional[Callable] = None,
450
+ renormalize: bool = True,
451
override_config: Optional[Dict[str, Any]] = None,
452
use_fp8: bool = False,
453
w1_scale: Optional[torch.Tensor] = None,
@@ -497,8 +498,12 @@ def fused_marlin_moe(hidden_states: torch.Tensor,
498
E = w1.shape[0]
499
N = w2.shape[1] * 16
500
500
- topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
501
- renormalize)
501
+ if custom_routing_function is None:
502
+ topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
503
+ renormalize)
504
+ else:
505
+ topk_weights, topk_ids = custom_routing_function(
506
+ hidden_states, gating_output, topk, renormalize)
507
508
get_config_func = functools.partial(try_get_optimal_moe_config,
509
w1.shape,
@@ -695,6 +700,7 @@ def fused_moe(
700
use_grouped_topk: bool = False,
701
num_expert_group: Optional[int] = None,
702
topk_group: Optional[int] = None,
703
+ custom_routing_function: Optional[Callable] = None,
704
use_fp8_w8a8: bool = False,
705
use_int8_w8a16: bool = False,
706
w1_scale: Optional[torch.Tensor] = None,
@@ -742,9 +748,12 @@ def fused_moe(
748
topk_weights, topk_ids = grouped_topk(hidden_states, gating_output,
749
topk, renormalize,
750
num_expert_group, topk_group)
745
- else:
751
+ elif custom_routing_function is None:
752
topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
753
renormalize)
754
+ else:
755
+ topk_weights, topk_ids = custom_routing_function(
756
+ hidden_states, gating_output, topk, renormalize)
757
758
return fused_experts(hidden_states,
759
w1,
vllm/model_executor/layers/fused_moe/layer.py
+56
-34
@@ -1,6 +1,6 @@
1
from abc import abstractmethod
2
from enum import Enum
3
-from typing import List, Optional, Tuple
3
+from typing import Callable, List, Optional, Tuple
4
5
import torch
6
@@ -62,15 +62,18 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
62
layer.register_parameter("w2_weight", w2_weight)
63
set_weight_attrs(w2_weight, extra_weight_attrs)
64
65
- def apply(self,
66
- layer: torch.nn.Module,
67
- x: torch.Tensor,
68
- router_logits: torch.Tensor,
69
- top_k: int,
70
- renormalize: bool,
71
- use_grouped_topk: bool,
72
- topk_group: Optional[int] = None,
73
- num_expert_group: Optional[int] = None) -> torch.Tensor:
65
+ def apply(
66
+ self,
67
+ layer: torch.nn.Module,
68
+ x: torch.Tensor,
69
+ router_logits: torch.Tensor,
70
+ top_k: int,
71
+ renormalize: bool,
72
+ use_grouped_topk: bool,
73
+ topk_group: Optional[int] = None,
74
+ num_expert_group: Optional[int] = None,
75
+ custom_routing_function: Optional[Callable] = None
76
+ ) -> torch.Tensor:
77
78
return self.forward(x=x,
79
layer=layer,
@@ -79,17 +82,21 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
82
renormalize=renormalize,
83
use_grouped_topk=use_grouped_topk,
84
topk_group=topk_group,
82
- num_expert_group=num_expert_group)
83
-
84
- def forward_cuda(self,
85
- layer: torch.nn.Module,
86
- x: torch.Tensor,
87
- use_grouped_topk: bool,
88
- top_k: int,
89
- router_logits: torch.Tensor,
90
- renormalize: bool,
91
- topk_group: Optional[int] = None,
92
- num_expert_group: Optional[int] = None) -> torch.Tensor:
85
+ num_expert_group=num_expert_group,
86
+ custom_routing_function=custom_routing_function)
87
+
88
+ def forward_cuda(
89
+ self,
90
+ layer: torch.nn.Module,
91
+ x: torch.Tensor,
92
+ use_grouped_topk: bool,
93
+ top_k: int,
94
+ router_logits: torch.Tensor,
95
+ renormalize: bool,
96
+ topk_group: Optional[int] = None,
97
+ num_expert_group: Optional[int] = None,
98
+ custom_routing_function: Optional[Callable] = None
99
+ ) -> torch.Tensor:
100
101
from vllm.model_executor.layers.fused_moe.fused_moe import (
102
fused_experts)
@@ -101,7 +108,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
108
top_k=top_k,
109
renormalize=renormalize,
110
topk_group=topk_group,
104
- num_expert_group=num_expert_group)
111
+ num_expert_group=num_expert_group,
112
+ custom_routing_function=custom_routing_function)
113
114
return fused_experts(hidden_states=x,
115
w1=layer.w13_weight,
@@ -114,20 +122,24 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
122
raise NotImplementedError(
123
"The CPU backend currently does not support MoE.")
124
117
- def forward_tpu(self,
118
- layer: torch.nn.Module,
119
- x: torch.Tensor,
120
- use_grouped_topk: bool,
121
- top_k: int,
122
- router_logits: torch.Tensor,
123
- renormalize: bool,
124
- topk_group: Optional[int] = None,
125
- num_expert_group: Optional[int] = None) -> torch.Tensor:
125
+ def forward_tpu(
126
+ self,
127
+ layer: torch.nn.Module,
128
+ x: torch.Tensor,
129
+ use_grouped_topk: bool,
130
+ top_k: int,
131
+ router_logits: torch.Tensor,
132
+ renormalize: bool,
133
+ topk_group: Optional[int] = None,
134
+ num_expert_group: Optional[int] = None,
135
+ custom_routing_function: Optional[Callable] = None
136
+ ) -> torch.Tensor:
137
138
from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe
139
assert not use_grouped_topk
140
assert num_expert_group is None
141
assert topk_group is None
142
+ assert custom_routing_function is None
143
return fused_moe(hidden_states=x,
144
w1=layer.w13_weight,
145
w2=layer.w2_weight,
@@ -172,6 +184,7 @@ class FusedMoE(torch.nn.Module):
184
quant_config: Optional[QuantizationConfig] = None,
185
tp_size: Optional[int] = None,
186
prefix: str = "",
187
+ custom_routing_function: Optional[Callable] = None,
188
):
189
super().__init__()
190
@@ -190,6 +203,7 @@ class FusedMoE(torch.nn.Module):
203
assert num_expert_group is not None and topk_group is not None
204
self.num_expert_group = num_expert_group
205
self.topk_group = topk_group
206
+ self.custom_routing_function = custom_routing_function
207
208
if quant_config is None:
209
self.quant_method: Optional[QuantizeMethodBase] = (
@@ -390,7 +404,8 @@ class FusedMoE(torch.nn.Module):
404
use_grouped_topk: bool,
405
renormalize: bool,
406
topk_group: Optional[int] = None,
393
- num_expert_group: Optional[int] = None):
407
+ num_expert_group: Optional[int] = None,
408
+ custom_routing_function: Optional[Callable] = None):
409
from vllm.model_executor.layers.fused_moe.fused_moe import (
410
fused_topk, grouped_topk)
411
@@ -405,11 +420,17 @@ class FusedMoE(torch.nn.Module):
420
renormalize=renormalize,
421
num_expert_group=num_expert_group,
422
topk_group=topk_group)
408
- else:
423
+ elif custom_routing_function is None:
424
topk_weights, topk_ids = fused_topk(hidden_states=hidden_states,
425
gating_output=router_logits,
426
topk=top_k,
427
renormalize=renormalize)
428
+ else:
429
+ topk_weights, topk_ids = custom_routing_function(
430
+ hidden_states=hidden_states,
431
+ gating_output=router_logits,
432
+ topk=top_k,
433
+ renormalize=renormalize)
434
435
return topk_weights, topk_ids
436
@@ -426,7 +447,8 @@ class FusedMoE(torch.nn.Module):
447
renormalize=self.renormalize,
448
use_grouped_topk=self.use_grouped_topk,
449
topk_group=self.topk_group,
429
- num_expert_group=self.num_expert_group)
450
+ num_expert_group=self.num_expert_group,
451
+ custom_routing_function=self.custom_routing_function)
452
453
if self.reduce_results and self.tp_size > 1:
454
final_hidden_states = tensor_model_parallel_all_reduce(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+14
-10
@@ -1,6 +1,6 @@
1
import enum
2
from enum import Enum
3
-from typing import List, Optional
3
+from typing import Callable, List, Optional
4
5
import torch
6
@@ -256,15 +256,18 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
256
)
257
replace_tensor("w2_weight_scale", marlin_w2_scales)
258
259
- def apply(self,
260
- layer: torch.nn.Module,
261
- x: torch.Tensor,
262
- router_logits: torch.Tensor,
263
- top_k: int,
264
- renormalize: bool = True,
265
- use_grouped_topk: bool = False,
266
- num_expert_group: Optional[int] = None,
267
- topk_group: Optional[int] = None) -> torch.Tensor:
259
+ def apply(
260
+ self,
261
+ layer: torch.nn.Module,
262
+ x: torch.Tensor,
263
+ router_logits: torch.Tensor,
264
+ top_k: int,
265
+ renormalize: bool = True,
266
+ use_grouped_topk: bool = False,
267
+ num_expert_group: Optional[int] = None,
268
+ topk_group: Optional[int] = None,
269
+ custom_routing_function: Optional[Callable] = None,
270
+ ) -> torch.Tensor:
271
272
from vllm.model_executor.layers.fused_moe.fused_moe import (
273
fused_marlin_moe)
@@ -278,6 +281,7 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
281
layer.w13_g_idx_sort_indices,
282
layer.w2_g_idx_sort_indices,
283
top_k,
284
+ custom_routing_function,
285
renormalize=renormalize,
286
w1_scale=layer.w13_weight_scale,
287
w2_scale=layer.w2_weight_scale)
vllm/model_executor/layers/quantization/experts_int8.py
+15
-11
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List, Optional
1
+from typing import Any, Callable, Dict, List, Optional
2
3
import torch
4
@@ -96,15 +96,18 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
96
requires_grad=False)
97
layer.register_parameter("w2_scale", w2_scale)
98
99
- def apply(self,
100
- layer: torch.nn.Module,
101
- x: torch.Tensor,
102
- router_logits: torch.Tensor,
103
- top_k: int,
104
- renormalize: bool = True,
105
- use_grouped_topk: bool = False,
106
- num_expert_group: Optional[int] = None,
107
- topk_group: Optional[int] = None) -> torch.Tensor:
99
+ def apply(
100
+ self,
101
+ layer: torch.nn.Module,
102
+ x: torch.Tensor,
103
+ router_logits: torch.Tensor,
104
+ top_k: int,
105
+ renormalize: bool = True,
106
+ use_grouped_topk: bool = False,
107
+ num_expert_group: Optional[int] = None,
108
+ topk_group: Optional[int] = None,
109
+ custom_routing_function: Optional[Callable] = None,
110
+ ) -> torch.Tensor:
111
from vllm.model_executor.layers.fused_moe import fused_experts
112
113
topk_weights, topk_ids = FusedMoE.select_experts(
@@ -114,7 +117,8 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
117
top_k=top_k,
118
renormalize=renormalize,
119
topk_group=topk_group,
117
- num_expert_group=num_expert_group)
120
+ num_expert_group=num_expert_group,
121
+ custom_routing_function=custom_routing_function)
122
123
return fused_experts(x,
124
layer.w13_weight,
vllm/model_executor/layers/quantization/fp8.py
+15
-11
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List, Optional
1
+from typing import Any, Callable, Dict, List, Optional
2
3
import torch
4
from torch.nn import Module
@@ -468,15 +468,18 @@ class Fp8MoEMethod(FusedMoEMethodBase):
468
requires_grad=False)
469
return
470
471
- def apply(self,
472
- layer: torch.nn.Module,
473
- x: torch.Tensor,
474
- router_logits: torch.Tensor,
475
- top_k: int,
476
- renormalize: bool,
477
- use_grouped_topk: bool,
478
- topk_group: Optional[int] = None,
479
- num_expert_group: Optional[int] = None) -> torch.Tensor:
471
+ def apply(
472
+ self,
473
+ layer: torch.nn.Module,
474
+ x: torch.Tensor,
475
+ router_logits: torch.Tensor,
476
+ top_k: int,
477
+ renormalize: bool,
478
+ use_grouped_topk: bool,
479
+ topk_group: Optional[int] = None,
480
+ num_expert_group: Optional[int] = None,
481
+ custom_routing_function: Optional[Callable] = None,
482
+ ) -> torch.Tensor:
483
484
from vllm.model_executor.layers.fused_moe import fused_experts
485
@@ -487,7 +490,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
490
top_k=top_k,
491
renormalize=renormalize,
492
topk_group=topk_group,
490
- num_expert_group=num_expert_group)
493
+ num_expert_group=num_expert_group,
494
+ custom_routing_function=custom_routing_function)
495
496
return fused_experts(x,
497
layer.w13_weight,
vllm/model_executor/layers/rotary_embedding.py
+15
-11
@@ -503,8 +503,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
503
dtype: torch.dtype,
504
short_factor: List[float],
505
long_factor: List[float],
506
- short_mscale: float = 1.0,
507
- long_mscale: float = 1.0,
506
+ short_mscale: Optional[float] = None,
507
+ long_mscale: Optional[float] = None,
508
):
509
super().__init__()
510
@@ -523,18 +523,22 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
523
self.base = base
524
self.short_factor = short_factor
525
self.long_factor = long_factor
526
- self.short_mscale = short_mscale
527
- self.long_mscale = long_mscale
528
-
529
- scale = (self.max_position_embeddings /
530
- self.original_max_position_embeddings)
526
527
+ scale = self.max_position_embeddings / \
528
+ self.original_max_position_embeddings
529
if scale <= 1.0:
533
- self.scaling_factor = 1.0
530
+ scaling_factor = 1.0
531
else:
535
- self.scaling_factor = math.sqrt(
532
+ scaling_factor = math.sqrt(
533
1 + math.log(scale) /
534
math.log(self.original_max_position_embeddings))
535
+ if short_mscale is None:
536
+ short_mscale = scaling_factor
537
+ if long_mscale is None:
538
+ long_mscale = scaling_factor
539
+
540
+ self.short_mscale = short_mscale
541
+ self.long_mscale = long_mscale
542
543
short_cache = self._compute_cos_sin_cache(
544
original_max_position_embeddings, short_factor, short_mscale)
@@ -571,8 +575,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
575
inv_freq = self._compute_inv_freq(rescale_factors)
576
t = torch.arange(max_position_embeddings, dtype=torch.float)
577
freqs = torch.einsum("i,j -> ij", t, inv_freq)
574
- cos = freqs.cos() * mscale * self.scaling_factor
575
- sin = freqs.sin() * mscale * self.scaling_factor
578
+ cos = freqs.cos() * mscale
579
+ sin = freqs.sin() * mscale
580
cache = torch.cat((cos, sin), dim=-1)
581
return cache
582
vllm/model_executor/models/__init__.py
+1
@@ -50,6 +50,7 @@ _GENERATION_MODELS = {
50
"PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
51
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
52
"Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
53
+ "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
54
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
55
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
56
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
vllm/model_executor/models/phimoe.py
new
+620
@@ -0,0 +1,620 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only PhiMoE model."""
24
+from typing import Iterable, List, Optional, Tuple
25
+
26
+import torch
27
+from torch import nn
28
+from transformers.configuration_utils import PretrainedConfig
29
+
30
+from vllm.attention import Attention, AttentionMetadata
31
+from vllm.config import CacheConfig, LoRAConfig
32
+from vllm.distributed import get_tensor_model_parallel_world_size
33
+from vllm.model_executor.layers.fused_moe import FusedMoE
34
+from vllm.model_executor.layers.linear import (QKVParallelLinear,
35
+ ReplicatedLinear,
36
+ RowParallelLinear)
37
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
+from vllm.model_executor.layers.quantization.base_config import (
39
+ QuantizationConfig)
40
+from vllm.model_executor.layers.rotary_embedding import get_rope
41
+from vllm.model_executor.layers.sampler import Sampler
42
+from vllm.model_executor.layers.vocab_parallel_embedding import (
43
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
44
+from vllm.model_executor.model_loader.weight_utils import (
45
+ default_weight_loader, maybe_remap_kv_scale_name)
46
+from vllm.model_executor.sampling_metadata import SamplingMetadata
47
+from vllm.sequence import IntermediateTensors, SamplerOutput
48
+
49
+from .interfaces import SupportsLoRA
50
+
51
+
52
+class PhiMoEConfig(PretrainedConfig):
53
+
54
+ model_type = "phimoe"
55
+ keys_to_ignore_at_inference = ["past_key_values"]
56
+
57
+ def __init__(
58
+ self,
59
+ vocab_size=32000,
60
+ hidden_size=4096,
61
+ intermediate_size=14336,
62
+ num_hidden_layers=32,
63
+ num_attention_heads=32,
64
+ num_key_value_heads=8,
65
+ hidden_act="silu",
66
+ max_position_embeddings=4096 * 32,
67
+ initializer_range=0.02,
68
+ rms_norm_eps=1e-5,
69
+ use_cache=True,
70
+ pad_token_id=None,
71
+ bos_token_id=1,
72
+ eos_token_id=2,
73
+ tie_word_embeddings=False,
74
+ rope_theta=1e6,
75
+ sliding_window=None,
76
+ attention_dropout=0.0,
77
+ num_experts_per_tok=2,
78
+ num_local_experts=16,
79
+ output_router_logits=False,
80
+ router_aux_loss_coef=0.001,
81
+ router_jitter_noise=0.0,
82
+ attention_bias=False,
83
+ lm_head_bias=False,
84
+ **kwargs,
85
+ ):
86
+ self.vocab_size = vocab_size
87
+ self.max_position_embeddings = max_position_embeddings
88
+ self.hidden_size = hidden_size
89
+ self.intermediate_size = intermediate_size
90
+ self.num_hidden_layers = num_hidden_layers
91
+ self.num_attention_heads = num_attention_heads
92
+ self.sliding_window = sliding_window
93
+ self.attention_bias = attention_bias
94
+ self.lm_head_bias = lm_head_bias
95
+ # for backward compatibility
96
+ if num_key_value_heads is None:
97
+ num_key_value_heads = num_attention_heads
98
+
99
+ self.num_key_value_heads = num_key_value_heads
100
+ self.hidden_act = hidden_act
101
+ self.initializer_range = initializer_range
102
+ self.rms_norm_eps = rms_norm_eps
103
+ self.use_cache = use_cache
104
+ self.rope_theta = rope_theta
105
+ self.attention_dropout = attention_dropout
106
+
107
+ self.num_experts_per_tok = num_experts_per_tok
108
+ self.num_local_experts = num_local_experts
109
+ self.output_router_logits = output_router_logits
110
+ self.router_aux_loss_coef = router_aux_loss_coef
111
+ self.router_jitter_noise = router_jitter_noise
112
+ super().__init__(
113
+ pad_token_id=pad_token_id,
114
+ bos_token_id=bos_token_id,
115
+ eos_token_id=eos_token_id,
116
+ tie_word_embeddings=tie_word_embeddings,
117
+ **kwargs,
118
+ )
119
+
120
+
121
+class mp(torch.autograd.Function):
122
+
123
+ @staticmethod
124
+ def forward(
125
+ ctx,
126
+ scores: torch.Tensor,
127
+ multiplier: torch.Tensor,
128
+ selected_experts: torch.Tensor,
129
+ masked_gates: torch.Tensor,
130
+ mask_for_one: torch.Tensor,
131
+ ):
132
+ ctx.save_for_backward(multiplier, selected_experts, masked_gates)
133
+ return multiplier * mask_for_one
134
+
135
+ @staticmethod
136
+ def backward(
137
+ ctx,
138
+ grad_at_output: torch.Tensor,
139
+ ):
140
+ multiplier, selected_experts, masked_gates = ctx.saved_tensors
141
+
142
+ grad_at_output = grad_at_output * multiplier
143
+
144
+ grad_at_scores_expaned = masked_gates * grad_at_output.mul(-1)
145
+ grad_at_scores_expaned.scatter_add_(
146
+ dim=-1,
147
+ index=selected_experts,
148
+ src=grad_at_output,
149
+ )
150
+
151
+ return (
152
+ grad_at_scores_expaned,
153
+ None,
154
+ None,
155
+ None,
156
+ None,
157
+ )
158
+
159
+
160
+def sparsemixer(scores, jitter_eps=0.01):
161
+ ################ first expert ################
162
+
163
+ with torch.no_grad():
164
+ # compute mask for sparsity
165
+ mask_logits_threshold, max_ind = scores.max(dim=-1, keepdim=True)
166
+ factor = scores.abs().clamp(min=mask_logits_threshold)
167
+ mask_logits_threshold = (
168
+ (mask_logits_threshold - scores) / factor) > (2 * jitter_eps)
169
+
170
+ # apply mask
171
+ masked_gates = scores.masked_fill(mask_logits_threshold, float("-inf"))
172
+ selected_experts = max_ind
173
+
174
+ # compute scores for gradients
175
+ masked_gates = torch.softmax(masked_gates, dim=-1)
176
+ multiplier_o = masked_gates.gather(dim=-1, index=selected_experts)
177
+
178
+ multiplier = multiplier_o
179
+
180
+ # masked out first expert
181
+ masked_scores = torch.scatter(
182
+ scores,
183
+ -1,
184
+ selected_experts,
185
+ float("-inf"),
186
+ )
187
+ with torch.no_grad():
188
+ # compute mask for sparsity
189
+ mask_logits_threshold, max_ind = masked_scores.max(dim=-1,
190
+ keepdim=True)
191
+ factor = scores.abs().clamp(min=mask_logits_threshold)
192
+ mask_logits_threshold = (
193
+ (mask_logits_threshold - scores) / factor) > (2 * jitter_eps)
194
+
195
+ # apply mask
196
+ masked_gates_top2 = masked_scores.masked_fill(mask_logits_threshold,
197
+ float("-inf"))
198
+ selected_experts_top2 = max_ind
199
+ # compute scores for gradients
200
+ masked_gates_top2 = torch.softmax(masked_gates_top2, dim=-1)
201
+ multiplier_top2 = masked_gates_top2.gather(dim=-1,
202
+ index=selected_experts_top2)
203
+
204
+ multiplier = torch.concat((multiplier, multiplier_top2), dim=-1)
205
+ selected_experts = torch.concat((selected_experts, selected_experts_top2),
206
+ dim=-1)
207
+
208
+ return (
209
+ multiplier,
210
+ selected_experts,
211
+ )
212
+
213
+
214
+def phimoe_routing_function(
215
+ hidden_states: torch.Tensor,
216
+ gating_output: torch.Tensor,
217
+ topk: int,
218
+ renormalize: bool,
219
+):
220
+ assert hidden_states.shape[0] == gating_output.shape[0], (
221
+ "Number of tokens mismatch")
222
+ assert topk == 2, "Only top-2 routing is supported"
223
+ assert renormalize is False, "Renormalization is not supported"
224
+
225
+ topk_weights, topk_ids = sparsemixer(gating_output)
226
+ return topk_weights, topk_ids
227
+
228
+
229
+class PhiMoE(nn.Module):
230
+ """A tensor-parallel MoE implementation for PhiMoE that shards each expert
231
+ across all ranks.
232
+
233
+ Each expert's weights are sharded across all ranks and a fused MoE
234
+ kernel is used for the forward pass, and finally we reduce the outputs
235
+ across ranks.
236
+ """
237
+
238
+ def __init__(
239
+ self,
240
+ num_experts: int,
241
+ top_k: int,
242
+ hidden_size: int,
243
+ intermediate_size: int,
244
+ params_dtype: Optional[torch.dtype] = None,
245
+ quant_config: Optional[QuantizationConfig] = None,
246
+ tp_size: Optional[int] = None,
247
+ ):
248
+ super().__init__()
249
+ self.hidden_size = hidden_size
250
+
251
+ # Gate always runs at half / full precision for now.
252
+ self.gate = ReplicatedLinear(
253
+ hidden_size,
254
+ num_experts,
255
+ bias=False,
256
+ params_dtype=params_dtype,
257
+ quant_config=None,
258
+ )
259
+
260
+ self.experts = FusedMoE(
261
+ num_experts=num_experts,
262
+ top_k=top_k,
263
+ hidden_size=hidden_size,
264
+ intermediate_size=intermediate_size,
265
+ params_dtype=params_dtype,
266
+ reduce_results=True,
267
+ renormalize=False,
268
+ quant_config=quant_config,
269
+ tp_size=tp_size,
270
+ custom_routing_function=phimoe_routing_function)
271
+
272
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
273
+ # NOTE: hidden_states can have either 1D or 2D shape.
274
+ orig_shape = hidden_states.shape
275
+ hidden_states = hidden_states.view(-1, self.hidden_size)
276
+ # router_logits: (num_tokens, n_experts)
277
+ router_logits, _ = self.gate(hidden_states)
278
+ final_hidden_states = self.experts(hidden_states, router_logits)
279
+ return final_hidden_states.view(orig_shape)
280
+
281
+
282
+class PhiMoEAttention(nn.Module):
283
+
284
+ def __init__(
285
+ self,
286
+ hidden_size: int,
287
+ num_heads: int,
288
+ num_kv_heads: int,
289
+ max_position: int = 4096 * 32,
290
+ rope_theta: float = 10000,
291
+ cache_config: Optional[CacheConfig] = None,
292
+ quant_config: Optional[QuantizationConfig] = None,
293
+ rope_scaling: Optional[dict] = None,
294
+ ) -> None:
295
+ super().__init__()
296
+ self.hidden_size = hidden_size
297
+ tp_size = get_tensor_model_parallel_world_size()
298
+ self.total_num_heads = num_heads
299
+ assert self.total_num_heads % tp_size == 0
300
+ self.num_heads = self.total_num_heads // tp_size
301
+ self.total_num_kv_heads = num_kv_heads
302
+ if self.total_num_kv_heads >= tp_size:
303
+ # Number of KV heads is greater than TP size, so we partition
304
+ # the KV heads across multiple tensor parallel GPUs.
305
+ assert self.total_num_kv_heads % tp_size == 0
306
+ else:
307
+ # Number of KV heads is less than TP size, so we replicate
308
+ # the KV heads across multiple tensor parallel GPUs.
309
+ assert tp_size % self.total_num_kv_heads == 0
310
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
311
+ self.head_dim = hidden_size // self.total_num_heads
312
+ self.q_size = self.num_heads * self.head_dim
313
+ self.kv_size = self.num_kv_heads * self.head_dim
314
+ self.scaling = self.head_dim**-0.5
315
+ self.rope_theta = rope_theta
316
+ self.rope_scaling = rope_scaling
317
+
318
+ self.qkv_proj = QKVParallelLinear(
319
+ hidden_size,
320
+ self.head_dim,
321
+ self.total_num_heads,
322
+ self.total_num_kv_heads,
323
+ bias=True,
324
+ quant_config=None,
325
+ )
326
+ self.o_proj = RowParallelLinear(
327
+ self.total_num_heads * self.head_dim,
328
+ hidden_size,
329
+ bias=True,
330
+ quant_config=None,
331
+ )
332
+ self.rotary_emb = get_rope(
333
+ self.head_dim,
334
+ rotary_dim=self.head_dim,
335
+ max_position=max_position,
336
+ base=int(self.rope_theta),
337
+ is_neox_style=True,
338
+ rope_scaling=self.rope_scaling,
339
+ )
340
+ self.attn = Attention(
341
+ self.num_heads,
342
+ self.head_dim,
343
+ self.scaling,
344
+ num_kv_heads=self.num_kv_heads,
345
+ cache_config=cache_config,
346
+ quant_config=quant_config,
347
+ )
348
+
349
+ def forward(
350
+ self,
351
+ positions: torch.Tensor,
352
+ hidden_states: torch.Tensor,
353
+ kv_cache: torch.Tensor,
354
+ attn_metadata: AttentionMetadata,
355
+ ) -> torch.Tensor:
356
+ qkv, _ = self.qkv_proj(hidden_states)
357
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
358
+ q, k = self.rotary_emb(positions, q, k)
359
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
360
+ output, _ = self.o_proj(attn_output)
361
+ return output
362
+
363
+
364
+class PhiMoEDecoderLayer(nn.Module):
365
+
366
+ def __init__(
367
+ self,
368
+ config: PhiMoEConfig,
369
+ cache_config: Optional[CacheConfig] = None,
370
+ quant_config: Optional[QuantizationConfig] = None,
371
+ ) -> None:
372
+ super().__init__()
373
+ self.hidden_size = config.hidden_size
374
+ # Requires transformers > 4.32.0
375
+ rope_theta = getattr(config, "rope_theta", 10000)
376
+ self.self_attn = PhiMoEAttention(
377
+ hidden_size=self.hidden_size,
378
+ num_heads=config.num_attention_heads,
379
+ max_position=config.max_position_embeddings,
380
+ num_kv_heads=config.num_key_value_heads,
381
+ rope_theta=rope_theta,
382
+ cache_config=cache_config,
383
+ quant_config=quant_config,
384
+ rope_scaling=config.rope_scaling,
385
+ )
386
+ self.block_sparse_moe = PhiMoE(
387
+ num_experts=config.num_local_experts,
388
+ top_k=config.num_experts_per_tok,
389
+ hidden_size=config.hidden_size,
390
+ intermediate_size=config.intermediate_size,
391
+ quant_config=quant_config,
392
+ )
393
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
394
+ eps=config.rms_norm_eps,
395
+ elementwise_affine=True)
396
+ self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
397
+ eps=config.rms_norm_eps,
398
+ elementwise_affine=True)
399
+
400
+ def forward(
401
+ self,
402
+ positions: torch.Tensor,
403
+ hidden_states: torch.Tensor,
404
+ kv_cache: torch.Tensor,
405
+ attn_metadata: AttentionMetadata,
406
+ residual: Optional[torch.Tensor],
407
+ ) -> torch.Tensor:
408
+ residual = hidden_states
409
+
410
+ # Self Attention
411
+ hidden_states = self.input_layernorm(hidden_states)
412
+
413
+ hidden_states = self.self_attn(
414
+ positions=positions,
415
+ hidden_states=hidden_states,
416
+ kv_cache=kv_cache,
417
+ attn_metadata=attn_metadata,
418
+ )
419
+ hidden_states = hidden_states + residual
420
+
421
+ # Fully Connected
422
+ residual = hidden_states
423
+ hidden_states = self.post_attention_layernorm(hidden_states)
424
+ hidden_states = self.block_sparse_moe(hidden_states)
425
+
426
+ hidden_states = hidden_states + residual
427
+ return hidden_states, residual
428
+
429
+
430
+class PhiMoEModel(nn.Module):
431
+
432
+ def __init__(
433
+ self,
434
+ config: PhiMoEConfig,
435
+ cache_config: Optional[CacheConfig] = None,
436
+ quant_config: Optional[QuantizationConfig] = None,
437
+ lora_config: Optional[LoRAConfig] = None,
438
+ ) -> None:
439
+ super().__init__()
440
+ self.padding_idx = config.pad_token_id
441
+ lora_vocab = ((lora_config.lora_extra_vocab_size *
442
+ (lora_config.max_loras or 1)) if lora_config else 0)
443
+ self.vocab_size = config.vocab_size + lora_vocab
444
+ self.org_vocab_size = config.vocab_size
445
+
446
+ self.embed_tokens = VocabParallelEmbedding(
447
+ self.vocab_size,
448
+ config.hidden_size,
449
+ org_num_embeddings=config.vocab_size,
450
+ )
451
+ self.layers = nn.ModuleList([
452
+ PhiMoEDecoderLayer(config, cache_config, quant_config=quant_config)
453
+ for _ in range(config.num_hidden_layers)
454
+ ])
455
+ self.norm = nn.LayerNorm(config.hidden_size,
456
+ eps=config.rms_norm_eps,
457
+ elementwise_affine=True)
458
+
459
+ def forward(
460
+ self,
461
+ input_ids: torch.Tensor,
462
+ positions: torch.Tensor,
463
+ kv_caches: List[torch.Tensor],
464
+ attn_metadata: AttentionMetadata,
465
+ ) -> torch.Tensor:
466
+ hidden_states = self.embed_tokens(input_ids)
467
+ residual = None
468
+ for i in range(len(self.layers)):
469
+ layer = self.layers[i]
470
+ hidden_states, residual = layer(positions, hidden_states,
471
+ kv_caches[i], attn_metadata,
472
+ residual)
473
+ hidden_states = self.norm(hidden_states)
474
+ return hidden_states
475
+
476
+
477
+class PhiMoEForCausalLM(nn.Module, SupportsLoRA):
478
+ fall_back_to_pt_during_load = False
479
+
480
+ packed_modules_mapping = {
481
+ "qkv_proj": [
482
+ "q_proj",
483
+ "k_proj",
484
+ "v_proj",
485
+ ],
486
+ }
487
+
488
+ # LoRA specific attributes
489
+ supported_lora_modules = [
490
+ "qkv_proj",
491
+ "o_proj",
492
+ "embed_tokens",
493
+ "lm_head",
494
+ ]
495
+ embedding_modules = {
496
+ "embed_tokens": "input_embeddings",
497
+ "lm_head": "output_embeddings",
498
+ }
499
+ embedding_padding_modules = ["lm_head"]
500
+
501
+ def __init__(
502
+ self,
503
+ config: PhiMoEConfig,
504
+ cache_config: Optional[CacheConfig] = None,
505
+ quant_config: Optional[QuantizationConfig] = None,
506
+ lora_config: Optional[LoRAConfig] = None,
507
+ ) -> None:
508
+ super().__init__()
509
+
510
+ self.config = config
511
+ self.lora_config = lora_config
512
+
513
+ self.model = PhiMoEModel(config,
514
+ cache_config,
515
+ quant_config,
516
+ lora_config=lora_config)
517
+ self.unpadded_vocab_size = config.vocab_size
518
+ if lora_config:
519
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
520
+ self.lm_head = ParallelLMHead(
521
+ self.unpadded_vocab_size,
522
+ config.hidden_size,
523
+ org_num_embeddings=config.vocab_size,
524
+ padding_size=(
525
+ DEFAULT_VOCAB_PADDING_SIZE
526
+ # We need bigger padding if using lora for kernel
527
+ # compatibility
528
+ if not lora_config else lora_config.lora_vocab_padding_size),
529
+ quant_config=None,
530
+ bias=True,
531
+ )
532
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
533
+ config.vocab_size)
534
+ self.sampler = Sampler()
535
+
536
+ def forward(
537
+ self,
538
+ input_ids: torch.Tensor,
539
+ positions: torch.Tensor,
540
+ kv_caches: List[torch.Tensor],
541
+ attn_metadata: AttentionMetadata,
542
+ intermediate_tensors: Optional[IntermediateTensors] = None,
543
+ ) -> torch.Tensor:
544
+ hidden_states = self.model(input_ids, positions, kv_caches,
545
+ attn_metadata)
546
+ return hidden_states
547
+
548
+ def compute_logits(self, hidden_states: torch.Tensor,
549
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
550
+ logits = self.logits_processor(self.lm_head, hidden_states,
551
+ sampling_metadata)
552
+ return logits
553
+
554
+ def sample(
555
+ self,
556
+ logits: Optional[torch.Tensor],
557
+ sampling_metadata: SamplingMetadata,
558
+ ) -> Optional[SamplerOutput]:
559
+ next_tokens = self.sampler(logits, sampling_metadata)
560
+ return next_tokens
561
+
562
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
563
+ stacked_params_mapping = [
564
+ # (param_name, shard_name, shard_id)
565
+ ("qkv_proj", "q_proj", "q"),
566
+ ("qkv_proj", "k_proj", "k"),
567
+ ("qkv_proj", "v_proj", "v"),
568
+ ]
569
+
570
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
571
+ ckpt_gate_proj_name="w1",
572
+ ckpt_down_proj_name="w2",
573
+ ckpt_up_proj_name="w3",
574
+ num_experts=self.config.num_local_experts)
575
+
576
+ params_dict = dict(self.named_parameters())
577
+ for name, loaded_weight in weights:
578
+ if "rotary_emb.inv_freq" in name:
579
+ continue
580
+
581
+ for param_name, weight_name, shard_id in stacked_params_mapping:
582
+ if weight_name not in name:
583
+ continue
584
+ name = name.replace(weight_name, param_name)
585
+ # Skip loading extra bias for GPTQ models.
586
+ if name.endswith(".bias") and name not in params_dict:
587
+ continue
588
+ param = params_dict[name]
589
+ weight_loader = param.weight_loader
590
+ weight_loader(param, loaded_weight, shard_id)
591
+ break
592
+ else:
593
+ for mapping in expert_params_mapping:
594
+ param_name, weight_name, expert_id, shard_id = mapping
595
+ if weight_name not in name:
596
+ continue
597
+ name = name.replace(weight_name, param_name)
598
+ param = params_dict[name]
599
+ weight_loader = param.weight_loader
600
+ weight_loader(
601
+ param,
602
+ loaded_weight,
603
+ weight_name,
604
+ shard_id=shard_id,
605
+ expert_id=expert_id,
606
+ )
607
+ break
608
+ else:
609
+ # Skip loading extra bias for GPTQ models.
610
+ if name.endswith(".bias") and name not in params_dict:
611
+ continue
612
+ # Remapping the name of FP8 kv-scale.
613
+ name = maybe_remap_kv_scale_name(name, params_dict)
614
+ if name is None:
615
+ continue
616
+
617
+ param = params_dict[name]
618
+ weight_loader = getattr(param, "weight_loader",
619
+ default_weight_loader)
620
+ weight_loader(param, loaded_weight)