A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Bugfix] Fix import of CutlassExpertsFp8 in compressed_tensors_moe.py (#20381)
Signed-off-by: Bill Nell <bnell@redhat.com>
bnellnm committed
Jul 2, 2025 at 22:07 UTC
2e25bb12a8f3b6fd8c2edcb5ae21c6c1cd315dbe
2 files changed
+11
-7
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+5
-3
@@ -14,9 +14,9 @@ import vllm.envs as envs
14
from vllm import _custom_ops as ops
15
from vllm.logger import init_logger
16
from vllm.model_executor.layers.fused_moe import (
17
- CutlassExpertsFp8, FusedMoE, FusedMoEActivationFormat, FusedMoEConfig,
18
- FusedMoEMethodBase, FusedMoEPermuteExpertsUnpermute,
19
- FusedMoEPrepareAndFinalize, FusedMoeWeightScaleSupported, fused_experts)
17
+ FusedMoE, FusedMoEActivationFormat, FusedMoEConfig, FusedMoEMethodBase,
18
+ FusedMoEPermuteExpertsUnpermute, FusedMoEPrepareAndFinalize,
19
+ FusedMoeWeightScaleSupported)
20
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa
21
WNA16_SUPPORTED_BITS, WNA16_SUPPORTED_TYPES_MAP)
22
from vllm.model_executor.layers.quantization.utils import replace_parameter
@@ -570,6 +570,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
570
del layer.w2_input_scale
571
self.fused_experts_func = None
572
else:
573
+ from vllm.model_executor.layers.fused_moe import fused_experts
574
self.fused_experts_func = fused_experts
575
576
def apply(
@@ -826,6 +827,7 @@ class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod):
827
prepare_finalize: FusedMoEPrepareAndFinalize,
828
moe: FusedMoEConfig,
829
) -> FusedMoEPermuteExpertsUnpermute:
830
+ from vllm.model_executor.layers.fused_moe import CutlassExpertsFp8
831
832
use_batched_format = (prepare_finalize.activation_format ==
833
FusedMoEActivationFormat.BatchedExperts)
vllm/model_executor/layers/quantization/fp8.py
+6
-4
@@ -14,10 +14,9 @@ from vllm import _custom_ops as ops
14
from vllm.distributed import get_tensor_model_parallel_world_size
15
from vllm.logger import init_logger
16
from vllm.model_executor.layers.fused_moe import (
17
- BatchedTritonOrDeepGemmExperts, FusedMoE, FusedMoEActivationFormat,
18
- FusedMoEConfig, FusedMoEMethodBase, FusedMoEPermuteExpertsUnpermute,
19
- FusedMoEPrepareAndFinalize, FusedMoeWeightScaleSupported,
20
- TritonOrDeepGemmExperts)
17
+ FusedMoE, FusedMoEActivationFormat, FusedMoEConfig, FusedMoEMethodBase,
18
+ FusedMoEPermuteExpertsUnpermute, FusedMoEPrepareAndFinalize,
19
+ FusedMoeWeightScaleSupported)
20
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
21
UnquantizedLinearMethod)
22
from vllm.model_executor.layers.quantization import QuantizationMethods
@@ -785,6 +784,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
784
prepare_finalize: FusedMoEPrepareAndFinalize,
785
moe: FusedMoEConfig,
786
) -> FusedMoEPermuteExpertsUnpermute:
787
+ from vllm.model_executor.layers.fused_moe import (
788
+ BatchedTritonOrDeepGemmExperts, TritonOrDeepGemmExperts)
789
+
790
assert not self.use_marlin and not self.rocm_aiter_moe_enabled, (
791
"Marlin and ROCm AITER are not supported with all2all yet.")
792