@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f4bc4de1b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core]refactor aqlm quant ops (#4351)

Kunshang Ji committed Apr 25, 2024 at 19:03 UTC f4bc4de1b1a1bd33bd3ec7dea3377eb75884250a
3 files changed +16 -2
benchmarks/kernels/benchmark_aqlm.py
+1 -1
@@ -6,7 +6,7 @@ from typing import Optional
6 import torch
7 import torch.nn.functional as F
8
9 -from vllm._C import ops
9 +from vllm import _custom_ops as ops
10 from vllm.model_executor.layers.quantization.aqlm import (
11 dequantize_weight, generic_dequantize_gemm, get_int_dtype,
12 optimized_dequantize_gemm)
vllm/_custom_ops.py
+14
@@ -153,6 +153,20 @@ def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
153 size_n, size_k)
154
155
156 +# aqlm
157 +def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
158 + codebooks: torch.Tensor, scales: torch.Tensor,
159 + codebook_partition_sizes: torch.Tensor,
160 + bias: Optional[torch.Tensor]) -> torch.Tensor:
161 + return vllm_ops.aqlm_gemm(input, codes, codebooks, scales,
162 + codebook_partition_sizes, bias)
163 +
164 +
165 +def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
166 + codebook_partition_sizes: torch.Tensor) -> torch.Tensor:
167 + return vllm_ops.aqlm_dequant(codes, codebooks, codebook_partition_sizes)
168 +
169 +
170 # fp8
171 def scaled_fp8_quant(input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
172 scale = torch.zeros(1, device=input.device, dtype=torch.float32)
vllm/model_executor/layers/quantization/aqlm.py
+1 -1
@@ -8,7 +8,7 @@ import torch
8 import torch.nn.functional as F
9 from torch.nn.parameter import Parameter
10
11 -from vllm._C import ops
11 +from vllm import _custom_ops as ops
12 from vllm.model_executor.layers.linear import (LinearMethodBase,
13 set_weight_attrs)
14 from vllm.model_executor.layers.quantization.base_config import (