A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core]refactor aqlm quant ops (#4351)
Kunshang Ji committed
Apr 25, 2024 at 19:03 UTC
f4bc4de1b1a1bd33bd3ec7dea3377eb75884250a
3 files changed
+16
-2
benchmarks/kernels/benchmark_aqlm.py
+1
-1
@@ -6,7 +6,7 @@ from typing import Optional
6
import torch
7
import torch.nn.functional as F
8
9
-from vllm._C import ops
9
+from vllm import _custom_ops as ops
10
from vllm.model_executor.layers.quantization.aqlm import (
11
dequantize_weight, generic_dequantize_gemm, get_int_dtype,
12
optimized_dequantize_gemm)
vllm/_custom_ops.py
+14
@@ -153,6 +153,20 @@ def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
153
size_n, size_k)
154
155
156
+# aqlm
157
+def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
158
+ codebooks: torch.Tensor, scales: torch.Tensor,
159
+ codebook_partition_sizes: torch.Tensor,
160
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
161
+ return vllm_ops.aqlm_gemm(input, codes, codebooks, scales,
162
+ codebook_partition_sizes, bias)
163
+
164
+
165
+def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
166
+ codebook_partition_sizes: torch.Tensor) -> torch.Tensor:
167
+ return vllm_ops.aqlm_dequant(codes, codebooks, codebook_partition_sizes)
168
+
169
+
170
# fp8
171
def scaled_fp8_quant(input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
172
scale = torch.zeros(1, device=input.device, dtype=torch.float32)
vllm/model_executor/layers/quantization/aqlm.py
+1
-1
@@ -8,7 +8,7 @@ import torch
8
import torch.nn.functional as F
9
from torch.nn.parameter import Parameter
10
11
-from vllm._C import ops
11
+from vllm import _custom_ops as ops
12
from vllm.model_executor.layers.linear import (LinearMethodBase,
13
set_weight_attrs)
14
from vllm.model_executor.layers.quantization.base_config import (