A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BUGFIX ] fix undefined silu_and_mul_nvfp4_quant (#23929)
Signed-off-by: hongchao <hongchao@msh.team> Signed-off-by: Richard Zou <zou3519@gmail.com> Co-authored-by: hongchao <hongchao@msh.team> Co-authored-by: Richard Zou <zou3519@gmail.com> Co-authored-by: Richard Zou <zou3519@users.noreply.github.com>
yzds committed
Aug 30, 2025 at 00:36 UTC
0dc9532065c5f98952cb82d4c497e49ca09400bf
3 files changed
+7
-4
csrc/ops.h
+2
-2
@@ -130,8 +130,8 @@ void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
130
void silu_and_mul_quant(torch::Tensor& out, torch::Tensor& input,
131
torch::Tensor& scale);
132
133
-#ifndef USE_ROCM
134
-
133
+#if (defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100) || \
134
+ (defined(ENABLE_NVFP4_SM120) && ENABLE_NVFP4_SM120)
135
void silu_and_mul_nvfp4_quant(torch::Tensor& out,
136
torch::Tensor& output_block_scale,
137
torch::Tensor& input,
csrc/torch_bindings.cpp
+2
-1
@@ -115,7 +115,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
115
"silu_and_mul_quant(Tensor! result, Tensor input, Tensor scale) -> ()");
116
ops.impl("silu_and_mul_quant", torch::kCUDA, &silu_and_mul_quant);
117
118
-#ifndef USE_ROCM
118
+#if (defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100) || \
119
+ (defined(ENABLE_NVFP4_SM120) && ENABLE_NVFP4_SM120)
120
ops.def(
121
"silu_and_mul_nvfp4_quant(Tensor! result, Tensor! result_block_scale, "
122
"Tensor input, Tensor input_global_scale) -> ()");
vllm/compilation/fix_functionalization.py
+3
-1
@@ -97,7 +97,9 @@ class FixFunctionalizationPass(VllmInductorPass):
97
node,
98
mutated_args,
99
args=('result', 'input', 'scale'))
100
- elif at_target == torch.ops._C.silu_and_mul_nvfp4_quant.default:
100
+ elif hasattr(
101
+ torch.ops._C, "silu_and_mul_nvfp4_quant"
102
+ ) and at_target == torch.ops._C.silu_and_mul_nvfp4_quant.default:
103
mutated_args = {1: 'result', 2: 'result_block_scale'}
104
self.defunctionalize(graph,
105
node,