A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 06:00 UTC
[Bugfix] Reject non-nvfp4 dtypes when using the flashinfer_nvlink_one_sided all2all backend (#39717)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Tyler Michael Smith committed
Apr 13, 2026 at 15:13 UTC
f72b20976c49f5cc522281b59df4863cabed8df2
1 file changed
+8
vllm/model_executor/layers/fused_moe/all2all_utils.py
+8
@@ -225,6 +225,14 @@ def maybe_make_prepare_finalize(
225
226
elif moe.use_fi_nvl_one_sided_kernels:
227
assert quant_config is not None
228
+ if quant_config.quant_dtype != "nvfp4":
229
+ raise ValueError(
230
+ "The 'flashinfer_nvlink_one_sided' all2all backend only "
231
+ "supports nvfp4 activation quantization, but got "
232
+ f"quant_dtype={quant_config.quant_dtype!r}. Use a different "
233
+ "all2all backend (e.g. 'flashinfer_nvlink_two_sided' or "
234
+ "'allgather_reducescatter') for non-nvfp4 models."
235
+ )
236
max_num_tokens = (
237
get_current_vllm_config().scheduler_config.max_num_batched_tokens
238
)