@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2acba47d9b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[bugfix] moe tuning. rm is_navi() (#12273)

Signed-off-by: Divakar Verma <divakar.verma@amd.com>

Divakar Verma committed Jan 21, 2025 at 16:47 UTC 2acba47d9bf97135d33355eff303d61a2c8d3d8a
1 file changed +2 -2
benchmarks/kernels/benchmark_moe.py
+2 -2
@@ -12,10 +12,10 @@ from transformers import AutoConfig
12
13 from vllm.model_executor.layers.fused_moe.fused_moe import *
14 from vllm.platforms import current_platform
15 -from vllm.utils import FlexibleArgumentParser, is_navi
15 +from vllm.utils import FlexibleArgumentParser
16
17 FP8_DTYPE = torch.float8_e4m3fnuz if current_platform.is_rocm(
18 -) and not is_navi() else torch.float8_e4m3fn
18 +) else torch.float8_e4m3fn
19
20
21 class BenchmarkConfig(TypedDict):