@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 32176fee73
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] support moe models (#9632)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Oct 27, 2024 at 21:58 UTC 32176fee733b76b295346870d717d44cb7102944
12 files changed +217 -78
benchmarks/kernels/benchmark_moe.py
+17 -16
@@ -88,22 +88,23 @@ def benchmark_config(
88 input_gating.copy_(gating_output[i])
89
90 def run():
91 - fused_moe(
92 - x,
93 - w1,
94 - w2,
95 - input_gating,
96 - topk,
97 - renormalize=True,
98 - inplace=True,
99 - override_config=config,
100 - use_fp8_w8a8=use_fp8_w8a8,
101 - use_int8_w8a16=use_int8_w8a16,
102 - w1_scale=w1_scale,
103 - w2_scale=w2_scale,
104 - a1_scale=a1_scale,
105 - a2_scale=a2_scale,
106 - )
91 + from vllm.model_executor.layers.fused_moe import override_config
92 + with override_config(config):
93 + fused_moe(
94 + x,
95 + w1,
96 + w2,
97 + input_gating,
98 + topk,
99 + renormalize=True,
100 + inplace=True,
101 + use_fp8_w8a8=use_fp8_w8a8,
102 + use_int8_w8a16=use_int8_w8a16,
103 + w1_scale=w1_scale,
104 + w2_scale=w2_scale,
105 + a1_scale=a1_scale,
106 + a2_scale=a2_scale,
107 + )
108
109 # JIT compilation & warmup
110 run()
tests/compile/test_basic_correctness.py
+2 -2
@@ -13,11 +13,11 @@ from ..utils import compare_all_settings
13 @pytest.mark.parametrize(
14 "model, model_args, pp_size, tp_size, attn_backend, method, fullgraph",
15 [
16 - ("meta-llama/Llama-3.2-1B", [], 2, 2, "FLASH_ATTN", "generate", True),
16 + ("meta-llama/Llama-3.2-1B", [], 2, 2, "FLASHINFER", "generate", True),
17 ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples",
18 ["--quantization", "compressed-tensors"
19 ], 1, 1, "FLASH_ATTN", "generate", True),
20 - ("google/gemma-2-2b-it", [], 1, 2, "FLASHINFER", "generate", True),
20 + ("ibm/PowerMoE-3b", [], 1, 2, "FLASH_ATTN", "generate", True),
21 # TODO: add multi-modality test for llava
22 ("llava-hf/llava-1.5-7b-hf", [], 2, 1, "FLASHINFER", "generate", False)
23 ])
tests/kernels/test_awq_marlin.py
+10 -11
@@ -5,11 +5,10 @@ Run `pytest tests/kernels/test_awq_marlin.py`.
5 import pytest
6 import torch
7
8 +import vllm.model_executor.layers.fused_moe # noqa
9 from tests.kernels.utils import (compute_max_diff, stack_and_dev, torch_moe,
10 torch_moe_single)
11 from vllm import _custom_ops as ops
11 -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
12 - fused_marlin_moe, single_marlin_moe)
12 from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
13 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
14 awq_marlin_quantize)
@@ -81,7 +80,7 @@ def test_fused_marlin_moe_awq(
80 score = torch.randn((m, e), device="cuda", dtype=dtype)
81
82 topk_weights, topk_ids = fused_topk(a, score, topk, False)
84 - marlin_output = fused_marlin_moe(
83 + marlin_output = torch.ops.vllm.fused_marlin_moe(
84 a,
85 qweight1,
86 qweight2,
@@ -150,14 +149,14 @@ def test_single_marlin_moe_multiply_awq(
149
150 score = torch.randn((m, e), device="cuda", dtype=dtype)
151
153 - marlin_output = single_marlin_moe(a,
154 - qweight,
155 - scales,
156 - score,
157 - topk,
158 - renormalize=False,
159 - w_zeros=zp,
160 - num_bits=num_bits)
152 + marlin_output = torch.ops.vllm.single_marlin_moe(a,
153 + qweight,
154 + scales,
155 + score,
156 + topk,
157 + renormalize=False,
158 + w_zeros=zp,
159 + num_bits=num_bits)
160
161 torch_output = torch_moe_single(a, w_ref.transpose(1, 2), score, topk)
162
tests/kernels/test_moe.py
+3 -4
@@ -7,12 +7,11 @@ import torch
7 from transformers import MixtralConfig
8 from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
9
10 +import vllm.model_executor.layers.fused_moe # noqa
11 from tests.kernels.utils import (compute_max_diff, opcheck, stack_and_dev,
12 torch_moe, torch_moe_single)
13 from vllm import _custom_ops as ops
14 from vllm.model_executor.layers.fused_moe import fused_moe
14 -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
15 - fused_marlin_moe, single_marlin_moe)
15 from vllm.model_executor.layers.fused_moe.fused_moe import (
16 fused_topk, moe_align_block_size)
17 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
@@ -193,7 +192,7 @@ def test_fused_marlin_moe(
192 topk,
193 renormalize=False,
194 )
196 - marlin_output = fused_marlin_moe(
195 + marlin_output = torch.ops.vllm.fused_marlin_moe(
196 a,
197 qweight1,
198 qweight2,
@@ -309,7 +308,7 @@ def test_single_marlin_moe_multiply(
308 sort_indices = stack_and_dev(sort_indices_l)
309
310 score = torch.randn((m, e), device="cuda", dtype=dtype)
312 - marlin_output = single_marlin_moe(
311 + marlin_output = torch.ops.vllm.single_marlin_moe(
312 a,
313 qweight,
314 scales,
vllm/model_executor/layers/fused_moe/__init__.py
+24 -4
@@ -1,23 +1,43 @@
1 +from contextlib import contextmanager
2 +from typing import Any, Dict, Optional
3 +
4 from vllm.model_executor.layers.fused_moe.layer import (
5 FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
6 from vllm.triton_utils import HAS_TRITON
7
8 +_config: Optional[Dict[str, Any]] = None
9 +
10 +
11 +@contextmanager
12 +def override_config(config):
13 + global _config
14 + old_config = _config
15 + _config = config
16 + yield
17 + _config = old_config
18 +
19 +
20 +def get_config() -> Optional[Dict[str, Any]]:
21 + return _config
22 +
23 +
24 __all__ = [
25 "FusedMoE",
26 "FusedMoEMethodBase",
27 "FusedMoeWeightScaleSupported",
28 + "override_config",
29 + "get_config",
30 ]
31
32 if HAS_TRITON:
12 - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
13 - fused_marlin_moe, single_marlin_moe)
33 + # import to register the custom ops
34 + import vllm.model_executor.layers.fused_moe.fused_marlin_moe # noqa
35 + import vllm.model_executor.layers.fused_moe.fused_moe # noqa
36 from vllm.model_executor.layers.fused_moe.fused_moe import (
37 fused_experts, fused_moe, fused_topk, get_config_file_name,
38 grouped_topk)
39
40 __all__ += [
19 - "fused_marlin_moe",
20 - "single_marlin_moe",
41 "fused_moe",
42 "fused_topk",
43 "fused_experts",
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+42 -9
@@ -1,6 +1,6 @@
1 """Fused MoE utilities for GPTQ."""
2 import functools
3 -from typing import Any, Dict, Optional
3 +from typing import Optional
4
5 import torch
6
@@ -18,6 +18,7 @@ def get_scalar_type(num_bits: int, has_zp: bool):
18 return scalar_types.uint4b8 if num_bits == 4 else scalar_types.uint8b128
19
20
21 +@torch.library.custom_op("vllm::single_marlin_moe", mutates_args=[])
22 def single_marlin_moe(
23 hidden_states: torch.Tensor,
24 w: torch.Tensor,
@@ -28,7 +29,6 @@ def single_marlin_moe(
29 g_idx: Optional[torch.Tensor] = None,
30 sort_indices: Optional[torch.Tensor] = None,
31 w_zeros: Optional[torch.Tensor] = None,
31 - override_config: Optional[Dict[str, Any]] = None,
32 num_bits: int = 8,
33 is_k_full: bool = True,
34 ) -> torch.Tensor:
@@ -49,8 +49,6 @@ def single_marlin_moe(
49 - topk (int): The number of top-k experts to select.
50 - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
51 - w_zeros (Optional[torch.Tensor]): Optional zero points to be used for w.
52 - - override_config (Optional[Dict[str, Any]]): Optional override
53 - for the kernel configuration.
52 - num_bits (bool): The number of bits in expert weights quantization.
53
54 Returns:
@@ -79,7 +77,6 @@ def single_marlin_moe(
77 w.shape,
78 topk_ids.shape[1],
79 None,
82 - override_config=override_config,
80 is_marlin=True)
81 config = get_config_func(M)
82
@@ -122,6 +119,24 @@ def single_marlin_moe(
119 return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
120
121
122 +@single_marlin_moe.register_fake
123 +def _(
124 + hidden_states: torch.Tensor,
125 + w: torch.Tensor,
126 + scales: torch.Tensor,
127 + gating_output: torch.Tensor,
128 + topk: int,
129 + renormalize: bool,
130 + g_idx: Optional[torch.Tensor] = None,
131 + sort_indices: Optional[torch.Tensor] = None,
132 + w_zeros: Optional[torch.Tensor] = None,
133 + num_bits: int = 8,
134 + is_k_full: bool = True,
135 +) -> torch.Tensor:
136 + return torch.empty_like(hidden_states)
137 +
138 +
139 +@torch.library.custom_op("vllm::fused_marlin_moe", mutates_args=[])
140 def fused_marlin_moe(
141 hidden_states: torch.Tensor,
142 w1: torch.Tensor,
@@ -137,7 +152,6 @@ def fused_marlin_moe(
152 sort_indices2: Optional[torch.Tensor] = None,
153 w1_zeros: Optional[torch.Tensor] = None,
154 w2_zeros: Optional[torch.Tensor] = None,
140 - override_config: Optional[Dict[str, Any]] = None,
155 num_bits: int = 8,
156 is_k_full: bool = True,
157 ) -> torch.Tensor:
@@ -161,8 +175,6 @@ def fused_marlin_moe(
175 permutation.
176 - topk_weights (torch.Tensor): Top-k weights.
177 - topk_ids (torch.Tensor): Indices of topk-k elements.
164 - - override_config (Optional[Dict[str, Any]]): Optional override
165 - for the kernel configuration.
178 - w1_zeros (Optional[torch.Tensor]): Optional zero points to be used for w1.
179 - w2_zeros (Optional[torch.Tensor]): Optional zero points to be used for w2.
180 - num_bits (bool): The number of bits in expert weights quantization.
@@ -209,7 +221,6 @@ def fused_marlin_moe(
221 w2.shape,
222 topk_ids.shape[1],
223 None,
212 - override_config=override_config,
224 is_marlin=True,
225 )
226 config = get_config_func(M)
@@ -311,3 +322,25 @@ def fused_marlin_moe(
322
323 return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
324 dim=1)
325 +
326 +
327 +@fused_marlin_moe.register_fake
328 +def _(
329 + hidden_states: torch.Tensor,
330 + w1: torch.Tensor,
331 + w2: torch.Tensor,
332 + w1_scale: torch.Tensor,
333 + w2_scale: torch.Tensor,
334 + gating_output: torch.Tensor,
335 + topk_weights: torch.Tensor,
336 + topk_ids: torch.Tensor,
337 + g_idx1: Optional[torch.Tensor] = None,
338 + g_idx2: Optional[torch.Tensor] = None,
339 + sort_indices1: Optional[torch.Tensor] = None,
340 + sort_indices2: Optional[torch.Tensor] = None,
341 + w1_zeros: Optional[torch.Tensor] = None,
342 + w2_zeros: Optional[torch.Tensor] = None,
343 + num_bits: int = 8,
344 + is_k_full: bool = True,
345 +) -> torch.Tensor:
346 + return torch.empty_like(hidden_states)
vllm/model_executor/layers/fused_moe/fused_moe.py
+93 -7
@@ -358,9 +358,10 @@ def try_get_optimal_moe_config(
358 top_k: int,
359 dtype: Optional[str],
360 M: int,
361 - override_config: Optional[Dict[str, Any]] = None,
361 is_marlin: bool = False,
362 ):
363 + from vllm.model_executor.layers.fused_moe import get_config
364 + override_config = get_config()
365 if override_config:
366 config = override_config
367 else:
@@ -465,19 +466,109 @@ def get_config_dtype_str(dtype: torch.dtype,
466 return None
467
468
469 +@torch.library.custom_op("vllm::inplace_fused_experts",
470 + mutates_args=["hidden_states"])
471 +def inplace_fused_experts(hidden_states: torch.Tensor,
472 + w1: torch.Tensor,
473 + w2: torch.Tensor,
474 + topk_weights: torch.Tensor,
475 + topk_ids: torch.Tensor,
476 + use_fp8_w8a8: bool = False,
477 + use_int8_w8a16: bool = False,
478 + w1_scale: Optional[torch.Tensor] = None,
479 + w2_scale: Optional[torch.Tensor] = None,
480 + a1_scale: Optional[torch.Tensor] = None,
481 + a2_scale: Optional[torch.Tensor] = None) -> None:
482 + fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids, True,
483 + use_fp8_w8a8, use_int8_w8a16, w1_scale, w2_scale,
484 + a1_scale, a2_scale)
485 +
486 +
487 +@inplace_fused_experts.register_fake
488 +def _(hidden_states: torch.Tensor,
489 + w1: torch.Tensor,
490 + w2: torch.Tensor,
491 + topk_weights: torch.Tensor,
492 + topk_ids: torch.Tensor,
493 + use_fp8_w8a8: bool = False,
494 + use_int8_w8a16: bool = False,
495 + w1_scale: Optional[torch.Tensor] = None,
496 + w2_scale: Optional[torch.Tensor] = None,
497 + a1_scale: Optional[torch.Tensor] = None,
498 + a2_scale: Optional[torch.Tensor] = None) -> None:
499 + pass
500 +
501 +
502 +@torch.library.custom_op("vllm::outplace_fused_experts", mutates_args=[])
503 +def outplace_fused_experts(
504 + hidden_states: torch.Tensor,
505 + w1: torch.Tensor,
506 + w2: torch.Tensor,
507 + topk_weights: torch.Tensor,
508 + topk_ids: torch.Tensor,
509 + use_fp8_w8a8: bool = False,
510 + use_int8_w8a16: bool = False,
511 + w1_scale: Optional[torch.Tensor] = None,
512 + w2_scale: Optional[torch.Tensor] = None,
513 + a1_scale: Optional[torch.Tensor] = None,
514 + a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
515 + return fused_experts_impl(hidden_states, w1, w2, topk_weights, topk_ids,
516 + False, use_fp8_w8a8, use_int8_w8a16, w1_scale,
517 + w2_scale, a1_scale, a2_scale)
518 +
519 +
520 +@outplace_fused_experts.register_fake
521 +def _(hidden_states: torch.Tensor,
522 + w1: torch.Tensor,
523 + w2: torch.Tensor,
524 + topk_weights: torch.Tensor,
525 + topk_ids: torch.Tensor,
526 + use_fp8_w8a8: bool = False,
527 + use_int8_w8a16: bool = False,
528 + w1_scale: Optional[torch.Tensor] = None,
529 + w2_scale: Optional[torch.Tensor] = None,
530 + a1_scale: Optional[torch.Tensor] = None,
531 + a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
532 + return torch.empty_like(hidden_states)
533 +
534 +
535 def fused_experts(hidden_states: torch.Tensor,
536 w1: torch.Tensor,
537 w2: torch.Tensor,
538 topk_weights: torch.Tensor,
539 topk_ids: torch.Tensor,
540 inplace: bool = False,
474 - override_config: Optional[Dict[str, Any]] = None,
541 use_fp8_w8a8: bool = False,
542 use_int8_w8a16: bool = False,
543 w1_scale: Optional[torch.Tensor] = None,
544 w2_scale: Optional[torch.Tensor] = None,
545 a1_scale: Optional[torch.Tensor] = None,
546 a2_scale: Optional[torch.Tensor] = None):
547 + if inplace:
548 + torch.ops.vllm.inplace_fused_experts(hidden_states, w1, w2,
549 + topk_weights, topk_ids,
550 + use_fp8_w8a8, use_int8_w8a16,
551 + w1_scale, w2_scale, a1_scale,
552 + a2_scale)
553 + return hidden_states
554 + else:
555 + return torch.ops.vllm.outplace_fused_experts(
556 + hidden_states, w1, w2, topk_weights, topk_ids, use_fp8_w8a8,
557 + use_int8_w8a16, w1_scale, w2_scale, a1_scale, a2_scale)
558 +
559 +
560 +def fused_experts_impl(hidden_states: torch.Tensor,
561 + w1: torch.Tensor,
562 + w2: torch.Tensor,
563 + topk_weights: torch.Tensor,
564 + topk_ids: torch.Tensor,
565 + inplace: bool = False,
566 + use_fp8_w8a8: bool = False,
567 + use_int8_w8a16: bool = False,
568 + w1_scale: Optional[torch.Tensor] = None,
569 + w2_scale: Optional[torch.Tensor] = None,
570 + a1_scale: Optional[torch.Tensor] = None,
571 + a2_scale: Optional[torch.Tensor] = None):
572 # Check constraints.
573 assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
574 assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
@@ -504,7 +595,6 @@ def fused_experts(hidden_states: torch.Tensor,
595 w2.shape,
596 topk_ids.shape[1],
597 config_dtype,
507 - override_config=override_config,
598 )
599
600 config = get_config_func(M)
@@ -602,7 +692,6 @@ def fused_moe(
692 topk: int,
693 renormalize: bool,
694 inplace: bool = False,
605 - override_config: Optional[Dict[str, Any]] = None,
695 use_grouped_topk: bool = False,
696 num_expert_group: Optional[int] = None,
697 topk_group: Optional[int] = None,
@@ -628,8 +717,6 @@ def fused_moe(
717 - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
718 - inplace (bool): If True, perform the operation in-place.
719 Defaults to False.
631 - - override_config (Optional[Dict[str, Any]]): Optional override
632 - for the kernel configuration.
720 - num_expert_group: Optional[int]: additional parameter for grouped_topk
721 - topk_group: Optional[int]: additional parameter for grouped_topk
722 - use_grouped_topk: If True, use grouped_topk instead of fused_topk
@@ -667,7 +754,6 @@ def fused_moe(
754 topk_weights,
755 topk_ids,
756 inplace=inplace,
670 - override_config=override_config,
757 use_fp8_w8a8=use_fp8_w8a8,
758 use_int8_w8a16=use_int8_w8a16,
759 w1_scale=w1_scale,
vllm/model_executor/layers/fused_moe/layer.py
+18 -11
@@ -12,7 +12,16 @@ from vllm.model_executor.custom_op import CustomOp
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
14 from vllm.model_executor.utils import set_weight_attrs
15 -
15 +from vllm.platforms import current_platform
16 +
17 +if current_platform.is_cuda_alike():
18 + from .fused_moe import fused_experts
19 +else:
20 + fused_experts = None # type: ignore
21 +if current_platform.is_tpu():
22 + from .moe_pallas import fused_moe as fused_moe_pallas
23 +else:
24 + fused_moe_pallas = None # type: ignore
25 logger = init_logger(__name__)
26
27
@@ -96,9 +105,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
105 num_expert_group: Optional[int] = None,
106 custom_routing_function: Optional[Callable] = None
107 ) -> torch.Tensor:
99 - from vllm.model_executor.layers.fused_moe.fused_moe import (
100 - fused_experts)
101 -
108 topk_weights, topk_ids = FusedMoE.select_experts(
109 hidden_states=x,
110 router_logits=router_logits,
@@ -132,17 +138,18 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
138 num_expert_group: Optional[int] = None,
139 custom_routing_function: Optional[Callable] = None
140 ) -> torch.Tensor:
135 - from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe
141 assert not use_grouped_topk
142 assert num_expert_group is None
143 assert topk_group is None
144 assert custom_routing_function is None
140 - return fused_moe(hidden_states=x,
141 - w1=layer.w13_weight,
142 - w2=layer.w2_weight,
143 - topk=top_k,
144 - gating_output=router_logits,
145 - renormalize=renormalize)
145 + return fused_moe_pallas(hidden_states=x,
146 + w1=layer.w13_weight,
147 + w2=layer.w2_weight,
148 + topk=top_k,
149 + gating_output=router_logits,
150 + renormalize=renormalize)
151 +
152 + forward_native = forward_cuda
153
154
155 class FusedMoE(torch.nn.Module):
vllm/model_executor/layers/quantization/awq_marlin.py
+2 -5
@@ -3,6 +3,7 @@ from typing import Any, Callable, Dict, List, Optional
3 import torch
4 from torch.nn import Parameter
5
6 +import vllm.model_executor.layers.fused_moe # noqa
7 from vllm import _custom_ops as ops
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.fused_moe.layer import (
@@ -435,10 +436,6 @@ class AWQMoEMethod(FusedMoEMethodBase):
436 topk_group: Optional[int] = None,
437 custom_routing_function: Optional[Callable] = None,
438 ) -> torch.Tensor:
438 -
439 - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
440 - fused_marlin_moe)
441 -
439 topk_weights, topk_ids = FusedMoE.select_experts(
440 hidden_states=x,
441 router_logits=router_logits,
@@ -449,7 +446,7 @@ class AWQMoEMethod(FusedMoEMethodBase):
446 num_expert_group=num_expert_group,
447 custom_routing_function=custom_routing_function)
448
452 - return fused_marlin_moe(
449 + return torch.ops.vllm.fused_marlin_moe(
450 x,
451 layer.w13_qweight,
452 layer.w2_qweight,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+2 -5
@@ -6,6 +6,7 @@ import torch
6 from compressed_tensors import CompressionFormat
7 from compressed_tensors.quantization import QuantizationStrategy
8
9 +import vllm.model_executor.layers.fused_moe # noqa
10 from vllm import _custom_ops as ops
11 from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
12 FusedMoeWeightScaleSupported)
@@ -481,10 +482,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
482 topk_group: Optional[int] = None,
483 custom_routing_function: Optional[Callable] = None,
484 ) -> torch.Tensor:
484 -
485 - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
486 - fused_marlin_moe)
487 -
485 topk_weights, topk_ids = FusedMoE.select_experts(
486 hidden_states=x,
487 router_logits=router_logits,
@@ -495,7 +492,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
492 num_expert_group=num_expert_group,
493 custom_routing_function=custom_routing_function)
494
498 - return fused_marlin_moe(
495 + return torch.ops.vllm.fused_marlin_moe(
496 x,
497 layer.w13_weight_packed,
498 layer.w2_weight_packed,
vllm/model_executor/layers/quantization/gptq_marlin.py
+2 -4
@@ -2,6 +2,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Union
2
3 import torch
4
5 +import vllm.model_executor.layers.fused_moe # noqa
6 from vllm import _custom_ops as ops
7 from vllm.logger import init_logger
8 from vllm.model_executor.layers.fused_moe.layer import (
@@ -536,9 +537,6 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
537 topk_group: Optional[int] = None,
538 custom_routing_function: Optional[Callable] = None,
539 ) -> torch.Tensor:
539 - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
540 - fused_marlin_moe)
541 -
540 # The input must currently be float16
541 orig_dtype = x.dtype
542 x = x.half()
@@ -553,7 +551,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
551 num_expert_group=num_expert_group,
552 custom_routing_function=None)
553
556 - return fused_marlin_moe(
554 + return torch.ops.vllm.fused_marlin_moe(
555 x,
556 layer.w13_qweight,
557 layer.w2_qweight,
vllm/model_executor/models/granitemoe.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28 from transformers.models.granitemoe import GraniteMoeConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -244,6 +245,7 @@ class GraniteMoeDecoderLayer(nn.Module):
245 return hidden_states
246
247
248 +@support_torch_compile
249 class GraniteMoeModel(nn.Module):
250
251 def __init__(