A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix][AMD] Update torch_bindings so that scaled_fp4_quant isn't build on ROCm (#13235)
Sage Moore committed
Feb 14, 2025 at 20:30 UTC
c9f9d5b397e6237d337efca81f7f9efb3f520859
3 files changed
+12
-10
csrc/ops.h
+4
-4
@@ -177,6 +177,10 @@ void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
177
std::optional<torch::Tensor> const& bias);
178
179
std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a);
180
+
181
+void scaled_fp4_quant(torch::Tensor& output, torch::Tensor const& input,
182
+ torch::Tensor& output_scale,
183
+ torch::Tensor const& input_scale);
184
#endif
185
186
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
@@ -194,10 +198,6 @@ torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
198
199
void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);
200
197
-void scaled_fp4_quant(torch::Tensor& output, torch::Tensor const& input,
198
- torch::Tensor& output_scale,
199
- torch::Tensor const& input_scale);
200
-
201
void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
202
torch::Tensor const& scale);
203
csrc/torch_bindings.cpp
+7
-6
@@ -385,6 +385,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
385
"bool silu_activation,"
386
"int pad_slot_id) -> ()");
387
ops.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd);
388
+
389
+ // Compute NVFP4 block quantized tensor.
390
+ ops.def(
391
+ "scaled_fp4_quant(Tensor! output, Tensor input,"
392
+ " Tensor! output_scale, Tensor input_scale) -> ()");
393
+ ops.impl("scaled_fp4_quant", torch::kCUDA, &scaled_fp4_quant);
394
+
395
#endif
396
397
// Quantized GEMM for GPTQ.
@@ -421,12 +428,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
428
ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
429
&dynamic_per_token_scaled_fp8_quant);
430
424
- // Compute NVFP4 block quantized tensor.
425
- ops.def(
426
- "scaled_fp4_quant(Tensor! output, Tensor input,"
427
- " Tensor! output_scale, Tensor input_scale) -> ()");
428
- ops.impl("scaled_fp4_quant", torch::kCUDA, &scaled_fp4_quant);
429
-
431
// Compute int8 quantized tensor for given scaling factor.
432
ops.def(
433
"static_scaled_int8_quant(Tensor! result, Tensor input, Tensor scale,"
vllm/_custom_ops.py
+1
@@ -774,6 +774,7 @@ def scaled_fp4_quant(
774
two values are packed into a uint8 and float8_e4m3 scaling factors
775
in the sizzled layout.
776
"""
777
+ assert not current_platform.is_rocm()
778
assert input.ndim >= 1, (
779
f'input.ndim needs to be >= 1, but got {input.ndim}.')
780
other_dims = 1 if input.ndim == 1 else -1