A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core]Refactor gptq_marlin ops (#4466)
Kunshang Ji committed
Apr 30, 2024 at 12:14 UTC
26f2fb51133c85ad8a57a87c8037f750dda757f4
2 files changed
+17
-1
vllm/_custom_ops.py
+16
@@ -167,6 +167,22 @@ def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
167
return vllm_ops.aqlm_dequant(codes, codebooks, codebook_partition_sizes)
168
169
170
+# gptq_marlin
171
+def gptq_marlin_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
172
+ size_k: int, size_n: int) -> torch.Tensor:
173
+ return vllm_ops.gptq_marlin_repack(b_q_weight, perm, size_k, size_n)
174
+
175
+
176
+def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
177
+ b_scales: torch.Tensor, g_idx: torch.Tensor,
178
+ perm: torch.Tensor, workspace: torch.Tensor, size_m: int,
179
+ size_n: int, size_k: int,
180
+ is_k_full: bool) -> torch.Tensor:
181
+ return vllm_ops.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
182
+ workspace, size_m, size_n, size_k,
183
+ is_k_full)
184
+
185
+
186
# fp8
187
def scaled_fp8_quant(
188
input: torch.Tensor,
vllm/model_executor/layers/quantization/gptq_marlin.py
+1
-1
@@ -6,7 +6,7 @@ import numpy
6
import torch
7
from torch.nn.parameter import Parameter
8
9
-from vllm._C import ops
9
+from vllm import _custom_ops as ops
10
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
set_weight_attrs)
12
from vllm.model_executor.layers.quantization.base_config import (