@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c9f9d5b397
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Bugfix][AMD] Update torch_bindings so that scaled_fp4_quant isn't build on ROCm (#13235)

Sage Moore committed Feb 14, 2025 at 20:30 UTC c9f9d5b397e6237d337efca81f7f9efb3f520859
3 files changed +12 -10
csrc/ops.h
+4 -4
@@ -177,6 +177,10 @@ void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
177 std::optional<torch::Tensor> const& bias);
178
179 std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a);
180 +
181 +void scaled_fp4_quant(torch::Tensor& output, torch::Tensor const& input,
182 + torch::Tensor& output_scale,
183 + torch::Tensor const& input_scale);
184 #endif
185
186 void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
@@ -194,10 +198,6 @@ torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
198
199 void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);
200
197 -void scaled_fp4_quant(torch::Tensor& output, torch::Tensor const& input,
198 - torch::Tensor& output_scale,
199 - torch::Tensor const& input_scale);
200 -
201 void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
202 torch::Tensor const& scale);
203
csrc/torch_bindings.cpp
+7 -6
@@ -385,6 +385,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
385 "bool silu_activation,"
386 "int pad_slot_id) -> ()");
387 ops.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd);
388 +
389 + // Compute NVFP4 block quantized tensor.
390 + ops.def(
391 + "scaled_fp4_quant(Tensor! output, Tensor input,"
392 + " Tensor! output_scale, Tensor input_scale) -> ()");
393 + ops.impl("scaled_fp4_quant", torch::kCUDA, &scaled_fp4_quant);
394 +
395 #endif
396
397 // Quantized GEMM for GPTQ.
@@ -421,12 +428,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
428 ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
429 &dynamic_per_token_scaled_fp8_quant);
430
424 - // Compute NVFP4 block quantized tensor.
425 - ops.def(
426 - "scaled_fp4_quant(Tensor! output, Tensor input,"
427 - " Tensor! output_scale, Tensor input_scale) -> ()");
428 - ops.impl("scaled_fp4_quant", torch::kCUDA, &scaled_fp4_quant);
429 -
431 // Compute int8 quantized tensor for given scaling factor.
432 ops.def(
433 "static_scaled_int8_quant(Tensor! result, Tensor input, Tensor scale,"
vllm/_custom_ops.py
+1
@@ -774,6 +774,7 @@ def scaled_fp4_quant(
774 two values are packed into a uint8 and float8_e4m3 scaling factors
775 in the sizzled layout.
776 """
777 + assert not current_platform.is_rocm()
778 assert input.ndim >= 1, (
779 f'input.ndim needs to be >= 1, but got {input.ndim}.')
780 other_dims = 1 if input.ndim == 1 else -1