A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 16:00 UTC
[ROCm][Bugfix] Fix ROCm runtime failure due to missing symbol (#38750)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com> Signed-off-by: Gregory Shtrasberg <156009573+gshtras@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: tjtanaavllm <tunjian.tan@amd.com>
Gregory Shtrasberg committed
Apr 1, 2026 at 23:30 UTC
3aab680e3e261e04c188b3015611f2947465d33b
2 files changed
+13
-12
csrc/ops.h
+2
@@ -142,11 +142,13 @@ void rms_norm_per_block_quant(torch::Tensor& out, torch::Tensor const& input,
142
std::optional<torch::Tensor> residual,
143
int64_t group_size, bool is_scale_transposed);
144
145
+#ifndef USE_ROCM
146
void silu_and_mul_per_block_quant(torch::Tensor& out,
147
torch::Tensor const& input,
148
torch::Tensor& scales, int64_t group_size,
149
std::optional<torch::Tensor> scale_ub,
150
bool is_scale_transposed);
151
+#endif
152
153
void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
154
std::optional<torch::Tensor> key, int64_t head_size,
csrc/torch_bindings.cpp
+11
-12
@@ -109,18 +109,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
109
"silu_and_mul_quant(Tensor! result, Tensor input, Tensor scale) -> ()");
110
ops.impl("silu_and_mul_quant", torch::kCUDA, &silu_and_mul_quant);
111
112
- // Fused SiLU+Mul + per-block quantization
113
- ops.def(
114
- "silu_and_mul_per_block_quant("
115
- "Tensor! out, "
116
- "Tensor input, "
117
- "Tensor! scales, "
118
- "int group_size, "
119
- "Tensor? scale_ub=None, "
120
- "bool is_scale_transposed=False) -> ()");
121
- ops.impl("silu_and_mul_per_block_quant", torch::kCUDA,
122
- &silu_and_mul_per_block_quant);
123
-
112
ops.def("mul_and_silu(Tensor! out, Tensor input) -> ()");
113
ops.impl("mul_and_silu", torch::kCUDA, &mul_and_silu);
114
@@ -244,6 +232,17 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
232
233
// Quantization ops
234
#ifndef USE_ROCM
235
+ // Fused SiLU+Mul + per-block quantization
236
+ ops.def(
237
+ "silu_and_mul_per_block_quant("
238
+ "Tensor! out, "
239
+ "Tensor input, "
240
+ "Tensor! scales, "
241
+ "int group_size, "
242
+ "Tensor? scale_ub=None, "
243
+ "bool is_scale_transposed=False) -> ()");
244
+ ops.impl("silu_and_mul_per_block_quant", torch::kCUDA,
245
+ &silu_and_mul_per_block_quant);
246
// DeepSeek V3 fused A GEMM (SM 9.0+, bf16 only, 1-16 tokens).
247
ops.def(
248
"dsv3_fused_a_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");