A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Bugfix] Fix imports for MoE on CPU (#15841)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Thien Tran committed
Apr 2, 2025 at 11:33 UTC
2039c6305bdcf2f920c80d8fcfce07cd6396f62f
1 file changed
+3
-2
vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py
+3
-2
@@ -4,8 +4,6 @@ from typing import List, Optional
4
import torch
5
6
import vllm.envs as envs
7
-from vllm.model_executor.layers.quantization.utils.fp8_utils import (
8
- per_token_group_quant_fp8)
7
from vllm.platforms import current_platform
8
9
@@ -38,6 +36,9 @@ def rocm_aiter_fused_experts(
36
import aiter as rocm_aiter
37
import aiter.fused_moe_bf16_asm as rocm_aiter_asm_fmoe
38
39
+ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
40
+ per_token_group_quant_fp8)
41
+
42
if envs.VLLM_ROCM_USE_AITER_FP8_BLOCK_SCALED_MOE and use_fp8_w8a8:
43
assert w1_scale is not None
44
assert w2_scale is not None