A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] support moe models (#9632)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Oct 27, 2024 at 21:58 UTC
32176fee733b76b295346870d717d44cb7102944
12 files changed
+217
-78
benchmarks/kernels/benchmark_moe.py
+17
-16
@@ -88,22 +88,23 @@ def benchmark_config(
88
input_gating.copy_(gating_output[i])
89
90
def run():
91
- fused_moe(
92
- x,
93
- w1,
94
- w2,
95
- input_gating,
96
- topk,
97
- renormalize=True,
98
- inplace=True,
99
- override_config=config,
100
- use_fp8_w8a8=use_fp8_w8a8,
101
- use_int8_w8a16=use_int8_w8a16,
102
- w1_scale=w1_scale,
103
- w2_scale=w2_scale,
104
- a1_scale=a1_scale,
105
- a2_scale=a2_scale,
106
- )
91
+ from vllm.model_executor.layers.fused_moe import override_config
92
+ with override_config(config):
93
+ fused_moe(
94
+ x,
95
+ w1,
96
+ w2,
97
+ input_gating,
98
+ topk,
99
+ renormalize=True,
100
+ inplace=True,
101
+ use_fp8_w8a8=use_fp8_w8a8,
102
+ use_int8_w8a16=use_int8_w8a16,
103
+ w1_scale=w1_scale,
104
+ w2_scale=w2_scale,
105
+ a1_scale=a1_scale,
106
+ a2_scale=a2_scale,
107
+ )
108
109
# JIT compilation & warmup
110
run()
tests/compile/test_basic_correctness.py
+2
-2
@@ -13,11 +13,11 @@ from ..utils import compare_all_settings
13
@pytest.mark.parametrize(
14
"model, model_args, pp_size, tp_size, attn_backend, method, fullgraph",
15
[
16
- ("meta-llama/Llama-3.2-1B", [], 2, 2, "FLASH_ATTN", "generate", True),
16
+ ("meta-llama/Llama-3.2-1B", [], 2, 2, "FLASHINFER", "generate", True),
17
("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples",
18
["--quantization", "compressed-tensors"
19
], 1, 1, "FLASH_ATTN", "generate", True),
20
- ("google/gemma-2-2b-it", [], 1, 2, "FLASHINFER", "generate", True),
20
+ ("ibm/PowerMoE-3b", [], 1, 2, "FLASH_ATTN", "generate", True),
21
# TODO: add multi-modality test for llava
22
("llava-hf/llava-1.5-7b-hf", [], 2, 1, "FLASHINFER", "generate", False)
23
])
tests/kernels/test_awq_marlin.py
+10
-11
@@ -5,11 +5,10 @@ Run `pytest tests/kernels/test_awq_marlin.py`.
5
import pytest
6
import torch
7
8
+import vllm.model_executor.layers.fused_moe # noqa
9
from tests.kernels.utils import (compute_max_diff, stack_and_dev, torch_moe,
10
torch_moe_single)
11
from vllm import _custom_ops as ops
11
-from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
12
- fused_marlin_moe, single_marlin_moe)
12
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
13
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
14
awq_marlin_quantize)
@@ -81,7 +80,7 @@ def test_fused_marlin_moe_awq(
80
score = torch.randn((m, e), device="cuda", dtype=dtype)
81
82
topk_weights, topk_ids = fused_topk(a, score, topk, False)
84
- marlin_output = fused_marlin_moe(
83
+ marlin_output = torch.ops.vllm.fused_marlin_moe(
84
a,
85
qweight1,
86
qweight2,
@@ -150,14 +149,14 @@ def test_single_marlin_moe_multiply_awq(
149
150
score = torch.randn((m, e), device="cuda", dtype=dtype)
151
153
- marlin_output = single_marlin_moe(a,
154
- qweight,
155
- scales,
156
- score,
157
- topk,
158
- renormalize=False,
159
- w_zeros=zp,
160
- num_bits=num_bits)
152
+ marlin_output = torch.ops.vllm.single_marlin_moe(a,
153
+ qweight,
154
+ scales,
155
+ score,
156
+ topk,
157
+ renormalize=False,
158
+ w_zeros=zp,
159
+ num_bits=num_bits)
160
161
torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
162
tests/kernels/test_moe.py
+3
-4
@@ -7,12 +7,11 @@ import torch
7
from transformers import MixtralConfig
8
from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
9
10
+import vllm.model_executor.layers.fused_moe # noqa
11
from tests.kernels.utils import (compute_max_diff, opcheck, stack_and_dev,
12
torch_moe, torch_moe_single)
13
from vllm import _custom_ops as ops
14
from vllm.model_executor.layers.fused_moe import fused_moe
14
-from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
15
- fused_marlin_moe, single_marlin_moe)
15
from vllm.model_executor.layers.fused_moe.fused_moe import (
16
fused_topk, moe_align_block_size)
17
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
@@ -193,7 +192,7 @@ def test_fused_marlin_moe(
192
topk,
193
renormalize=False,
194
)
196
- marlin_output = fused_marlin_moe(
195
+ marlin_output = torch.ops.vllm.fused_marlin_moe(
196
a,
197
qweight1,
198
qweight2,
@@ -309,7 +308,7 @@ def test_single_marlin_moe_multiply(
308
sort_indices = stack_and_dev(sort_indices_l)
309
310
score = torch.randn((m, e), device="cuda", dtype=dtype)
312
- marlin_output = single_marlin_moe(
311
+ marlin_output = torch.ops.vllm.single_marlin_moe(
312
a,
313
qweight,
314
scales,
vllm/model_executor/layers/fused_moe/__init__.py
+24
-4
@@ -1,23 +1,43 @@
1
+from contextlib import contextmanager
2
+from typing import Any, Dict, Optional
3
+
4
from vllm.model_executor.layers.fused_moe.layer import (
5
FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
6
from vllm.triton_utils import HAS_TRITON
7
8
+_config: Optional[Dict[str, Any]] = None
9
+
10
+
11
+@contextmanager
12
+def override_config(config):
13
+ global _config
14
+ old_config = _config
15
+ _config = config
16
+ yield
17
+ _config = old_config
18
+
19
+
20
+def get_config() -> Optional[Dict[str, Any]]:
21
+ return _config
22
+
23
+
24
__all__ = [
25
"FusedMoE",
26
"FusedMoEMethodBase",
27
"FusedMoeWeightScaleSupported",
28
+ "override_config",
29
+ "get_config",
30
]
31
32
if HAS_TRITON:
12
- from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
13
- fused_marlin_moe, single_marlin_moe)
33
+ # import to register the custom ops
34
+ import vllm.model_executor.layers.fused_moe.fused_marlin_moe # noqa
35
+ import vllm.model_executor.layers.fused_moe.fused_moe # noqa
36
from vllm.model_executor.layers.fused_moe.fused_moe import (
37
fused_experts, fused_moe, fused_topk, get_config_file_name,
38
grouped_topk)
39
40
__all__ += [
19
- "fused_marlin_moe",
20
- "single_marlin_moe",
41
"fused_moe",
42
"fused_topk",
43
"fused_experts",
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+42
-9
@@ -1,6 +1,6 @@
1
"""Fused MoE utilities for GPTQ."""
2
import functools
3
-from typing import Any, Dict, Optional
3
+from typing import Optional
4
5
import torch
6
@@ -18,6 +18,7 @@ def get_scalar_type(num_bits: int, has_zp: bool):
18
return scalar_types.uint4b8 if num_bits == 4 else scalar_types.uint8b128
19
20
21
+@torch.library.custom_op("vllm::single_marlin_moe", mutates_args=[])
22
def single_marlin_moe(
23
hidden_states: torch.Tensor,
24
w: torch.Tensor,
@@ -28,7 +29,6 @@ def single_marlin_moe(
29
g_idx: Optional[torch.Tensor] = None,
30
sort_indices: Optional[torch.Tensor] = None,
31
w_zeros: Optional[torch.Tensor] = None,
31
- override_config: Optional[Dict[str, Any]] = None,
32
num_bits: int = 8,
33
is_k_full: bool = True,
34
) -> torch.Tensor:
@@ -49,8 +49,6 @@ def single_marlin_moe(
49
- topk (int): The number of top-k experts to select.
50
- renormalize (bool): If True, renormalize the top-k weights to sum to 1.
51
- w_zeros (Optional[torch.Tensor]): Optional zero points to be used for w.
52
- - override_config (Optional[Dict[str, Any]]): Optional override
53
- for the kernel configuration.
52
- num_bits (bool): The number of bits in expert weights quantization.
53
54
Returns:
@@ -79,7 +77,6 @@ def single_marlin_moe(
77
w.shape,
78
topk_ids.shape[1],
79
None,
82
- override_config=override_config,
80
is_marlin=True)
81
config = get_config_func(M)
82
@@ -122,6 +119,24 @@ def single_marlin_moe(
119
return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
120
121
122
+@single_marlin_moe.register_fake
123
+def _(
124
+ hidden_states: torch.Tensor,
125
+ w: torch.Tensor,
126
+ scales: torch.Tensor,
127
+ gating_output: torch.Tensor,
128
+ topk: int,
129
+ renormalize: bool,
130
+ g_idx: Optional[torch.Tensor] = None,
131
+ sort_indices: Optional[torch.Tensor] = None,
132
+ w_zeros: Optional[torch.Tensor] = None,
133
+ num_bits: int = 8,
134
+ is_k_full: bool = True,
135
+) -> torch.Tensor:
136
+ return torch.empty_like(hidden_states)
137
+
138
+
139
+@torch.library.custom_op("vllm::fused_marlin_moe", mutates_args=[])
140
def fused_marlin_moe(
141
hidden_states: torch.Tensor,
142
w1: torch.Tensor,
@@ -137,7 +152,6 @@ def fused_marlin_moe(
152
sort_indices2: Optional[torch.Tensor] = None,
153
w1_zeros: Optional[torch.Tensor] = None,
154
w2_zeros: Optional[torch.Tensor] = None,
140
- override_config: Optional[Dict[str, Any]] = None,
155
num_bits: int = 8,
156
is_k_full: bool = True,
157
) -> torch.Tensor:
@@ -161,8 +175,6 @@ def fused_marlin_moe(
175
permutation.
176
- topk_weights (torch.Tensor): Top-k weights.
177
- topk_ids (torch.Tensor): Indices of topk-k elements.
164
- - override_config (Optional[Dict[str, Any]]): Optional override
165
- for the kernel configuration.
178
- w1_zeros (Optional[torch.Tensor]): Optional zero points to be used for w1.
179
- w2_zeros (Optional[torch.Tensor]): Optional zero points to be used for w2.
180
- num_bits (bool): The number of bits in expert weights quantization.
@@ -209,7 +221,6 @@ def fused_marlin_moe(
221
w2.shape,
222
topk_ids.shape[1],
223
None,
212
- override_config=override_config,
224
is_marlin=True,
225
)
226
config = get_config_func(M)
@@ -311,3 +322,25 @@ def fused_marlin_moe(
322
323
return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
324
dim=1)
325
+
326
+
327
+@fused_marlin_moe.register_fake
328
+def _(
329
+ hidden_states: torch.Tensor,
330
+ w1: torch.Tensor,
331
+ w2: torch.Tensor,
332
+ w1_scale: torch.Tensor,
333
+ w2_scale: torch.Tensor,
334
+ gating_output: torch.Tensor,
335
+ topk_weights: torch.Tensor,
336
+ topk_ids: torch.Tensor,
337
+ g_idx1: Optional[torch.Tensor] = None,
338
+ g_idx2: Optional[torch.Tensor] = None,
339
+ sort_indices1: Optional[torch.Tensor] = None,
340
+ sort_indices2: Optional[torch.Tensor] = None,
341
+ w1_zeros: Optional[torch.Tensor] = None,
342
+ w2_zeros: Optional[torch.Tensor] = None,
343
+ num_bits: int = 8,
344
+ is_k_full: bool = True,
345
+) -> torch.Tensor:
346
+ return torch.empty_like(hidden_states)
vllm/model_executor/layers/fused_moe/fused_moe.py
+93
-7
@@ -358,9 +358,10 @@ def try_get_optimal_moe_config(
358
top_k: int,
359
dtype: Optional[str],
360
M: int,
361
- override_config: Optional[Dict[str, Any]] = None,
361
is_marlin: bool = False,
362
):
363
+ from vllm.model_executor.layers.fused_moe import get_config
364
+ override_config = get_config()
365
if override_config:
366
config = override_config
367
else:
@@ -465,19 +466,109 @@ def get_config_dtype_str(dtype: torch.dtype,
466
return None
467
468
469
+@torch.library.custom_op("vllm::inplace_fused_experts",
470
+ mutates_args=["hidden_states"])
471
+def inplace_fused_experts(hidden_states: torch.Tensor,
472
+ w1: torch.Tensor,
473
+ w2: torch.Tensor,
474
+ topk_weights: torch.Tensor,
475
+ topk_ids: torch.Tensor,
476
+ use_fp8_w8a8: bool = False,
477
+ use_int8_w8a16: bool = False,
478
+ w1_scale: Optional[torch.Tensor] = None,
479
+ w2_scale: Optional[torch.Tensor] = None,
480
+ a1_scale: Optional[torch.Tensor] = None,
481
+ a2_scale: Optional[torch.Tensor] = None) -> None:
482
+ fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids, True,
483
+ use_fp8_w8a8, use_int8_w8a16, w1_scale, w2_scale,
484
+ a1_scale, a2_scale)
485
+
486
+
487
+@inplace_fused_experts.register_fake
488
+def _(hidden_states: torch.Tensor,
489
+ w1: torch.Tensor,
490
+ w2: torch.Tensor,
491
+ topk_weights: torch.Tensor,
492
+ topk_ids: torch.Tensor,
493
+ use_fp8_w8a8: bool = False,
494
+ use_int8_w8a16: bool = False,
495
+ w1_scale: Optional[torch.Tensor] = None,
496
+ w2_scale: Optional[torch.Tensor] = None,
497
+ a1_scale: Optional[torch.Tensor] = None,
498
+ a2_scale: Optional[torch.Tensor] = None) -> None:
499
+ pass
500
+
501
+
502
+@torch.library.custom_op("vllm::outplace_fused_experts", mutates_args=[])
503
+def outplace_fused_experts(
504
+ hidden_states: torch.Tensor,
505
+ w1: torch.Tensor,
506
+ w2: torch.Tensor,
507
+ topk_weights: torch.Tensor,
508
+ topk_ids: torch.Tensor,
509
+ use_fp8_w8a8: bool = False,
510
+ use_int8_w8a16: bool = False,
511
+ w1_scale: Optional[torch.Tensor] = None,
512
+ w2_scale: Optional[torch.Tensor] = None,
513
+ a1_scale: Optional[torch.Tensor] = None,
514
+ a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
515
+ return fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids,
516
+ False, use_fp8_w8a8, use_int8_w8a16, w1_scale,
517
+ w2_scale, a1_scale, a2_scale)
518
+
519
+
520
+@outplace_fused_experts.register_fake
521
+def _(hidden_states: torch.Tensor,
522
+ w1: torch.Tensor,
523
+ w2: torch.Tensor,
524
+ topk_weights: torch.Tensor,
525
+ topk_ids: torch.Tensor,
526
+ use_fp8_w8a8: bool = False,
527
+ use_int8_w8a16: bool = False,
528
+ w1_scale: Optional[torch.Tensor] = None,
529
+ w2_scale: Optional[torch.Tensor] = None,
530
+ a1_scale: Optional[torch.Tensor] = None,
531
+ a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
532
+ return torch.empty_like(hidden_states)
533
+
534
+
535
def fused_experts(hidden_states: torch.Tensor,
536
w1: torch.Tensor,
537
w2: torch.Tensor,
538
topk_weights: torch.Tensor,
539
topk_ids: torch.Tensor,
540
inplace: bool = False,
474
- override_config: Optional[Dict[str, Any]] = None,
541
use_fp8_w8a8: bool = False,
542
use_int8_w8a16: bool = False,
543
w1_scale: Optional[torch.Tensor] = None,
544
w2_scale: Optional[torch.Tensor] = None,
545
a1_scale: Optional[torch.Tensor] = None,
546
a2_scale: Optional[torch.Tensor] = None):
547
+ if inplace:
548
+ torch.ops.vllm.inplace_fused_experts(hidden_states, w1, w2,
549
+ topk_weights, topk_ids,
550
+ use_fp8_w8a8, use_int8_w8a16,
551
+ w1_scale, w2_scale, a1_scale,
552
+ a2_scale)
553
+ return hidden_states
554
+ else:
555
+ return torch.ops.vllm.outplace_fused_experts(
556
+ hidden_states, w1, w2, topk_weights, topk_ids, use_fp8_w8a8,
557
+ use_int8_w8a16, w1_scale, w2_scale, a1_scale, a2_scale)
558
+
559
+
560
+def fused_experts_impl(hidden_states: torch.Tensor,
561
+ w1: torch.Tensor,
562
+ w2: torch.Tensor,
563
+ topk_weights: torch.Tensor,
564
+ topk_ids: torch.Tensor,
565
+ inplace: bool = False,
566
+ use_fp8_w8a8: bool = False,
567
+ use_int8_w8a16: bool = False,
568
+ w1_scale: Optional[torch.Tensor] = None,
569
+ w2_scale: Optional[torch.Tensor] = None,
570
+ a1_scale: Optional[torch.Tensor] = None,
571
+ a2_scale: Optional[torch.Tensor] = None):
572
# Check constraints.
573
assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
574
assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
@@ -504,7 +595,6 @@ def fused_experts(hidden_states: torch.Tensor,
595
w2.shape,
596
topk_ids.shape[1],
597
config_dtype,
507
- override_config=override_config,
598
)
599
600
config = get_config_func(M)
@@ -602,7 +692,6 @@ def fused_moe(
692
topk: int,
693
renormalize: bool,
694
inplace: bool = False,
605
- override_config: Optional[Dict[str, Any]] = None,
695
use_grouped_topk: bool = False,
696
num_expert_group: Optional[int] = None,
697
topk_group: Optional[int] = None,
@@ -628,8 +717,6 @@ def fused_moe(
717
- renormalize (bool): If True, renormalize the top-k weights to sum to 1.
718
- inplace (bool): If True, perform the operation in-place.
719
Defaults to False.
631
- - override_config (Optional[Dict[str, Any]]): Optional override
632
- for the kernel configuration.
720
- num_expert_group: Optional[int]: additional parameter for grouped_topk
721
- topk_group: Optional[int]: additional parameter for grouped_topk
722
- use_grouped_topk: If True, use grouped_topk instead of fused_topk
@@ -667,7 +754,6 @@ def fused_moe(
754
topk_weights,
755
topk_ids,
756
inplace=inplace,
670
- override_config=override_config,
757
use_fp8_w8a8=use_fp8_w8a8,
758
use_int8_w8a16=use_int8_w8a16,
759
w1_scale=w1_scale,
vllm/model_executor/layers/fused_moe/layer.py
+18
-11
@@ -12,7 +12,16 @@ from vllm.model_executor.custom_op import CustomOp
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig, QuantizeMethodBase)
14
from vllm.model_executor.utils import set_weight_attrs
15
-
15
+from vllm.platforms import current_platform
16
+
17
+if current_platform.is_cuda_alike():
18
+ from .fused_moe import fused_experts
19
+else:
20
+ fused_experts = None # type: ignore
21
+if current_platform.is_tpu():
22
+ from .moe_pallas import fused_moe as fused_moe_pallas
23
+else:
24
+ fused_moe_pallas = None # type: ignore
25
logger = init_logger(__name__)
26
27
@@ -96,9 +105,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
105
num_expert_group: Optional[int] = None,
106
custom_routing_function: Optional[Callable] = None
107
) -> torch.Tensor:
99
- from vllm.model_executor.layers.fused_moe.fused_moe import (
100
- fused_experts)
101
-
108
topk_weights, topk_ids = FusedMoE.select_experts(
109
hidden_states=x,
110
router_logits=router_logits,
@@ -132,17 +138,18 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
138
num_expert_group: Optional[int] = None,
139
custom_routing_function: Optional[Callable] = None
140
) -> torch.Tensor:
135
- from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe
141
assert not use_grouped_topk
142
assert num_expert_group is None
143
assert topk_group is None
144
assert custom_routing_function is None
140
- return fused_moe(hidden_states=x,
141
- w1=layer.w13_weight,
142
- w2=layer.w2_weight,
143
- topk=top_k,
144
- gating_output=router_logits,
145
- renormalize=renormalize)
145
+ return fused_moe_pallas(hidden_states=x,
146
+ w1=layer.w13_weight,
147
+ w2=layer.w2_weight,
148
+ topk=top_k,
149
+ gating_output=router_logits,
150
+ renormalize=renormalize)
151
+
152
+ forward_native = forward_cuda
153
154
155
class FusedMoE(torch.nn.Module):
vllm/model_executor/layers/quantization/awq_marlin.py
+2
-5
@@ -3,6 +3,7 @@ from typing import Any, Callable, Dict, List, Optional
3
import torch
4
from torch.nn import Parameter
5
6
+import vllm.model_executor.layers.fused_moe # noqa
7
from vllm import _custom_ops as ops
8
from vllm.logger import init_logger
9
from vllm.model_executor.layers.fused_moe.layer import (
@@ -435,10 +436,6 @@ class AWQMoEMethod(FusedMoEMethodBase):
436
topk_group: Optional[int] = None,
437
custom_routing_function: Optional[Callable] = None,
438
) -> torch.Tensor:
438
-
439
- from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
440
- fused_marlin_moe)
441
-
439
topk_weights, topk_ids = FusedMoE.select_experts(
440
hidden_states=x,
441
router_logits=router_logits,
@@ -449,7 +446,7 @@ class AWQMoEMethod(FusedMoEMethodBase):
446
num_expert_group=num_expert_group,
447
custom_routing_function=custom_routing_function)
448
452
- return fused_marlin_moe(
449
+ return torch.ops.vllm.fused_marlin_moe(
450
x,
451
layer.w13_qweight,
452
layer.w2_qweight,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+2
-5
@@ -6,6 +6,7 @@ import torch
6
from compressed_tensors import CompressionFormat
7
from compressed_tensors.quantization import QuantizationStrategy
8
9
+import vllm.model_executor.layers.fused_moe # noqa
10
from vllm import _custom_ops as ops
11
from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
12
FusedMoeWeightScaleSupported)
@@ -481,10 +482,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
482
topk_group: Optional[int] = None,
483
custom_routing_function: Optional[Callable] = None,
484
) -> torch.Tensor:
484
-
485
- from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
486
- fused_marlin_moe)
487
-
485
topk_weights, topk_ids = FusedMoE.select_experts(
486
hidden_states=x,
487
router_logits=router_logits,
@@ -495,7 +492,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
492
num_expert_group=num_expert_group,
493
custom_routing_function=custom_routing_function)
494
498
- return fused_marlin_moe(
495
+ return torch.ops.vllm.fused_marlin_moe(
496
x,
497
layer.w13_weight_packed,
498
layer.w2_weight_packed,
vllm/model_executor/layers/quantization/gptq_marlin.py
+2
-4
@@ -2,6 +2,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Union
2
3
import torch
4
5
+import vllm.model_executor.layers.fused_moe # noqa
6
from vllm import _custom_ops as ops
7
from vllm.logger import init_logger
8
from vllm.model_executor.layers.fused_moe.layer import (
@@ -536,9 +537,6 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
537
topk_group: Optional[int] = None,
538
custom_routing_function: Optional[Callable] = None,
539
) -> torch.Tensor:
539
- from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
540
- fused_marlin_moe)
541
-
540
# The input must currently be float16
541
orig_dtype = x.dtype
542
x = x.half()
@@ -553,7 +551,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
551
num_expert_group=num_expert_group,
552
custom_routing_function=None)
553
556
- return fused_marlin_moe(
554
+ return torch.ops.vllm.fused_marlin_moe(
555
x,
556
layer.w13_qweight,
557
layer.w2_qweight,
vllm/model_executor/models/granitemoe.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28
from transformers.models.granitemoe import GraniteMoeConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -244,6 +245,7 @@ class GraniteMoeDecoderLayer(nn.Module):
245
return hidden_states
246
247
248
+@support_torch_compile
249
class GraniteMoeModel(nn.Module):
250
251
def __init__(