@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 26f2fb5113
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core]Refactor gptq_marlin ops (#4466)

Kunshang Ji committed Apr 30, 2024 at 12:14 UTC 26f2fb51133c85ad8a57a87c8037f750dda757f4
2 files changed +17 -1
vllm/_custom_ops.py
+16
@@ -167,6 +167,22 @@ def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
167 return vllm_ops.aqlm_dequant(codes, codebooks, codebook_partition_sizes)
168
169
170 +# gptq_marlin
171 +def gptq_marlin_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
172 + size_k: int, size_n: int) -> torch.Tensor:
173 + return vllm_ops.gptq_marlin_repack(b_q_weight, perm, size_k, size_n)
174 +
175 +
176 +def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
177 + b_scales: torch.Tensor, g_idx: torch.Tensor,
178 + perm: torch.Tensor, workspace: torch.Tensor, size_m: int,
179 + size_n: int, size_k: int,
180 + is_k_full: bool) -> torch.Tensor:
181 + return vllm_ops.gptq_marlin_gemm(a, b_q_weight, b_scales, g_idx, perm,
182 + workspace, size_m, size_n, size_k,
183 + is_k_full)
184 +
185 +
186 # fp8
187 def scaled_fp8_quant(
188 input: torch.Tensor,
vllm/model_executor/layers/quantization/gptq_marlin.py
+1 -1
@@ -6,7 +6,7 @@ import numpy
6 import torch
7 from torch.nn.parameter import Parameter
8
9 -from vllm._C import ops
9 +from vllm import _custom_ops as ops
10 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 set_weight_attrs)
12 from vllm.model_executor.layers.quantization.base_config import (