@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5cc6bddb6e

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Kernel] Add GPTQv2 format support for low-bit or asymmetric quantization, by adapting gptq_gemm (#26092)

Xiangyu Li committed Oct 24, 2025 at 11:26 UTC 5cc6bddb6ef5e8e5c10de8122a43fd6e8c1e3b4b

8 files changed +295 -98

csrc/ops.h

+1 -1

@@ -307,7 +307,7 @@ void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,

307 torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,

308 torch::Tensor b_gptq_qzeros,

309 torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,

310 - bool use_exllama, int64_t bit);

310 + bool use_exllama, bool use_v2_format, int64_t bit);

311

312 void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);

313

csrc/quantization/gptq/q_gemm.cu

+131 -92

@@ -185,7 +185,7 @@ typedef void (*fp_gemm_half_q_half_gptq_kernel)(const half*, const uint32_t*,

185 const uint32_t*, const half*,

186 half*, const int, const int,

187 const int, const int,

188 - const int*);

188 + const bool, const int*);

189

190 template <bool first_block, int m_count>

191 __global__ void gemm_half_q_half_gptq_4bit_kernel(

@@ -193,12 +193,15 @@ __global__ void gemm_half_q_half_gptq_4bit_kernel(

193 const uint32_t* __restrict__ b_gptq_qzeros,

194 const half* __restrict__ b_gptq_scales, half* __restrict__ c,

195 const int size_m, const int size_n, const int size_k, const int groups,

196 - const int* __restrict__ b_q_perm) {

196 + const bool use_v2_format, const int* __restrict__ b_q_perm) {

197 MatrixView_half a_(a, size_m, size_k);

198 MatrixView_half_rw c_(c, size_m, size_n);

199 MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);

200 MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);

201

202 + // GPTQv2 and GPTQv1 handles zero points differently

203 + int zero_offset = use_v2_format ? 0 : 1;

204 +

205 auto t = threadIdx.x;

206

207 // Block

@@ -256,10 +259,10 @@ __global__ void gemm_half_q_half_gptq_4bit_kernel(

259 half2 y1y16[4][2];

260 b_gptq_qzeros_.item4(zeros, group, n);

261 b_gptq_scales_.item4_f(scales, group, n);

259 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);

260 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);

261 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);

262 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);

262 + dequant_4bit_8_prep_zero(zeros[0] + zero_offset, z1z16[0], y1y16[0]);

263 + dequant_4bit_8_prep_zero(zeros[1] + zero_offset, z1z16[1], y1y16[1]);

264 + dequant_4bit_8_prep_zero(zeros[2] + zero_offset, z1z16[2], y1y16[2]);

265 + dequant_4bit_8_prep_zero(zeros[3] + zero_offset, z1z16[3], y1y16[3]);

266

267 // Column result

268 float block_c[m_count][4] = {};

@@ -272,10 +275,10 @@ __global__ void gemm_half_q_half_gptq_4bit_kernel(

275 nextgroup += groupsize;

276 b_gptq_qzeros_.item4(zeros, group, n);

277 b_gptq_scales_.item4_f(scales, group, n);

275 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);

276 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);

277 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);

278 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);

278 + dequant_4bit_8_prep_zero(zeros[0] + zero_offset, z1z16[0], y1y16[0]);

279 + dequant_4bit_8_prep_zero(zeros[1] + zero_offset, z1z16[1], y1y16[1]);

280 + dequant_4bit_8_prep_zero(zeros[2] + zero_offset, z1z16[2], y1y16[2]);

281 + dequant_4bit_8_prep_zero(zeros[3] + zero_offset, z1z16[3], y1y16[3]);

282 }

283

284 #pragma unroll

@@ -329,12 +332,15 @@ __global__ void gemm_half_q_half_gptq_2bit_kernel(

332 const uint32_t* __restrict__ b_gptq_qzeros,

333 const half* __restrict__ b_gptq_scales, half* __restrict__ c,

334 const int size_m, const int size_n, const int size_k, const int groups,

332 - const int* __restrict__ b_q_perm) {

335 + const bool use_v2_format, const int* __restrict__ b_q_perm) {

336 MatrixView_half a_(a, size_m, size_k);

337 MatrixView_half_rw c_(c, size_m, size_n);

338 MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);

339 MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);

340

341 + // GPTQv2 and GPTQv1 handles zero points differently

342 + int zero_offset = use_v2_format ? 0 : 1;

343 +

344 auto t = threadIdx.x;

345

346 // Block

@@ -409,10 +415,10 @@ __global__ void gemm_half_q_half_gptq_2bit_kernel(

415 int4 load_int4 = *b_ptr4;

416

417 half2 dq[4][8];

412 - dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);

413 - dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);

414 - dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);

415 - dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);

418 + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + zero_offset);

419 + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + zero_offset);

420 + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + zero_offset);

421 + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + zero_offset);

422

423 #pragma unroll

424 for (int m = 0; m < m_count; m++) {

@@ -448,12 +454,15 @@ __global__ void gemm_half_q_half_gptq_3bit_kernel(

454 const uint32_t* __restrict__ b_gptq_qzeros,

455 const half* __restrict__ b_gptq_scales, half* __restrict__ c,

456 const int size_m, const int size_n, const int size_k, const int groups,

451 - const int* __restrict__ b_q_perm) {

457 + const bool use_v2_format, const int* __restrict__ b_q_perm) {

458 MatrixView_half a_(a, size_m, size_k);

459 MatrixView_half_rw c_(c, size_m, size_n);

460 MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);

461 MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);

462

463 + // GPTQv2 and GPTQv1 handles zero points differently

464 + int zero_offset = use_v2_format ? 0 : 1;

465 +

466 auto t = threadIdx.x;

467

468 // Block

@@ -534,13 +543,13 @@ __global__ void gemm_half_q_half_gptq_3bit_kernel(

543

544 half2 dq[4][16];

545 dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0],

537 - size_n, zeros[0] + 1);

546 + size_n, zeros[0] + zero_offset);

547 dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1],

539 - size_n, zeros[1] + 1);

548 + size_n, zeros[1] + zero_offset);

549 dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2],

541 - size_n, zeros[2] + 1);

550 + size_n, zeros[2] + zero_offset);

551 dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3],

543 - size_n, zeros[3] + 1);

552 + size_n, zeros[3] + zero_offset);

553

554 #pragma unroll

555 for (int m = 0; m < m_count; m++) {

@@ -574,12 +583,15 @@ __global__ void gemm_half_q_half_gptq_8bit_kernel(

583 const uint32_t* __restrict__ b_gptq_qzeros,

584 const half* __restrict__ b_gptq_scales, half* __restrict__ c,

585 const int size_m, const int size_n, const int size_k, const int groups,

577 - const int* __restrict__ b_q_perm) {

586 + const bool use_v2_format, const int* __restrict__ b_q_perm) {

587 MatrixView_half a_(a, size_m, size_k);

588 MatrixView_half_rw c_(c, size_m, size_n);

589 MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);

590 MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);

591

592 + // GPTQv2 and GPTQv1 handles zero points differently

593 + int zero_offset = use_v2_format ? 0 : 1;

594 +

595 auto t = threadIdx.x;

596

597 // Block

@@ -658,13 +670,13 @@ __global__ void gemm_half_q_half_gptq_8bit_kernel(

670

671 half2 dq[4][4];

672 dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n,

661 - zeros[0] + 1);

673 + zeros[0] + zero_offset);

674 dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n,

663 - zeros[1] + 1);

675 + zeros[1] + zero_offset);

676 dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n,

665 - zeros[2] + 1);

677 + zeros[2] + zero_offset);

678 dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n,

667 - zeros[3] + 1);

679 + zeros[3] + zero_offset);

680

681 for (int m = 0; m < m_count; m++) {

682 block_c[m][0] =

@@ -730,7 +742,8 @@ void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,

742 const uint32_t* b_gptq_qzeros,

743 const half* b_gptq_scales, const int* b_q_perm,

744 half* c, int size_m, int size_n, int size_k,

733 - int m_count, int groups, int bit) {

745 + int m_count, int groups, bool use_v2_format,

746 + int bit) {

747 dim3 blockDim, gridDim;

748 blockDim.x = BLOCK_KN_SIZE;

749 blockDim.y = 1;

@@ -743,20 +756,23 @@ void gemm_half_q_half_cuda_part(const half* a, const uint32_t* b_q_weight,

756 pick_gemm_half_q_half_gptq_kernel(true, m_count, bit);

757

758 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();

746 - kernel<<<gridDim, blockDim, 0, stream>>>(a, b_q_weight, b_gptq_qzeros,

747 - b_gptq_scales, c, size_m, size_n,

748 - size_k, groups, b_q_perm);

759 + kernel<<<gridDim, blockDim, 0, stream>>>(

760 + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c, size_m, size_n, size_k,

761 + groups, use_v2_format, b_q_perm);

762 }

763

764 __global__ void reconstruct_exllama_8bit_kernel(

765 const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,

766 const uint32_t* __restrict__ b_gptq_qzeros,

767 const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,

755 - const int groups, half* __restrict__ b) {

768 + const int groups, const bool use_v2_format, half* __restrict__ b) {

769 MatrixView_half_rw b_(b, size_k, size_n);

770 MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);

771 MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);

772

773 + // GPTQv2 and GPTQv1 handles zero points differently

774 + int zero_offset = use_v2_format ? 0 : 1;

775 +

776 auto offset_k = BLOCK_KN_SIZE * blockIdx.y;

777 auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;

778

@@ -812,13 +828,13 @@ __global__ void reconstruct_exllama_8bit_kernel(

828

829 half2 dq[4][4];

830 dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n,

815 - zeros[0] + 1);

831 + zeros[0] + zero_offset);

832 dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n,

817 - zeros[1] + 1);

833 + zeros[1] + zero_offset);

834 dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n,

819 - zeros[2] + 1);

835 + zeros[2] + zero_offset);

836 dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n,

821 - zeros[3] + 1);

837 + zeros[3] + zero_offset);

838

839 // half* dqh = (half*)dq;

840 if (b_q_perm) {

@@ -849,11 +865,14 @@ __global__ void reconstruct_exllama_4bit_kernel(

865 const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,

866 const uint32_t* __restrict__ b_gptq_qzeros,

867 const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,

852 - const int groups, half* __restrict__ b) {

868 + const int groups, const bool use_v2_format, half* __restrict__ b) {

869 MatrixView_half_rw b_(b, size_k, size_n);

870 MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);

871 MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);

872

873 + // GPTQv2 and GPTQv1 handles zero points differently

874 + int zero_offset = use_v2_format ? 0 : 1;

875 +

876 auto offset_k = BLOCK_KN_SIZE * blockIdx.y;

877 auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;

878

@@ -888,10 +907,10 @@ __global__ void reconstruct_exllama_4bit_kernel(

907 half2 y1y16[4][2];

908 b_gptq_qzeros_.item4(zeros, group, n);

909 b_gptq_scales_.item4_h2(scales, group, n);

891 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);

892 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);

893 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);

894 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);

910 + dequant_4bit_8_prep_zero(zeros[0] + zero_offset, z1z16[0], y1y16[0]);

911 + dequant_4bit_8_prep_zero(zeros[1] + zero_offset, z1z16[1], y1y16[1]);

912 + dequant_4bit_8_prep_zero(zeros[2] + zero_offset, z1z16[2], y1y16[2]);

913 + dequant_4bit_8_prep_zero(zeros[3] + zero_offset, z1z16[3], y1y16[3]);

914

915 __syncthreads();

916

@@ -904,10 +923,10 @@ __global__ void reconstruct_exllama_4bit_kernel(

923 nextgroup += groupsize;

924 b_gptq_qzeros_.item4(zeros, group, n);

925 b_gptq_scales_.item4_h2(scales, group, n);

907 - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]);

908 - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]);

909 - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]);

910 - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]);

926 + dequant_4bit_8_prep_zero(zeros[0] + zero_offset, z1z16[0], y1y16[0]);

927 + dequant_4bit_8_prep_zero(zeros[1] + zero_offset, z1z16[1], y1y16[1]);

928 + dequant_4bit_8_prep_zero(zeros[2] + zero_offset, z1z16[2], y1y16[2]);

929 + dequant_4bit_8_prep_zero(zeros[3] + zero_offset, z1z16[3], y1y16[3]);

930 }

931

932 for (int p = 0; p < 4; p++) {

@@ -954,11 +973,14 @@ __global__ void reconstruct_exllama_3bit_kernel(

973 const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,

974 const uint32_t* __restrict__ b_gptq_qzeros,

975 const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,

957 - const int groups, half* __restrict__ b) {

976 + const int groups, const bool use_v2_format, half* __restrict__ b) {

977 MatrixView_half_rw b_(b, size_k, size_n);

978 MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);

979 MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);

980

981 + // GPTQv2 and GPTQv1 handles zero points differently

982 + int zero_offset = use_v2_format ? 0 : 1;

983 +

984 auto offset_k = BLOCK_KN_SIZE * blockIdx.y;

985 auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;

986

@@ -1016,13 +1038,13 @@ __global__ void reconstruct_exllama_3bit_kernel(

1038

1039 half2 dq[4][16];

1040 dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0],

1019 - size_n, zeros[0] + 1);

1041 + size_n, zeros[0] + zero_offset);

1042 dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1],

1021 - size_n, zeros[1] + 1);

1043 + size_n, zeros[1] + zero_offset);

1044 dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2],

1023 - size_n, zeros[2] + 1);

1045 + size_n, zeros[2] + zero_offset);

1046 dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3],

1025 - size_n, zeros[3] + 1);

1047 + size_n, zeros[3] + zero_offset);

1048

1049 if (b_q_perm) {

1050 for (int j = 0; j < 16; j++) {

@@ -1052,11 +1074,14 @@ __global__ void reconstruct_exllama_2bit_kernel(

1074 const uint32_t* __restrict__ b_q_weight, const int* __restrict__ b_q_perm,

1075 const uint32_t* __restrict__ b_gptq_qzeros,

1076 const half* __restrict__ b_gptq_scales, const int size_k, const int size_n,

1055 - const int groups, half* __restrict__ b) {

1077 + const int groups, const bool use_v2_format, half* __restrict__ b) {

1078 MatrixView_half_rw b_(b, size_k, size_n);

1079 MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);

1080 MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);

1081

1082 + // GPTQv2 and GPTQv1 handles zero points differently

1083 + int zero_offset = use_v2_format ? 0 : 1;

1084 +

1085 auto offset_k = BLOCK_KN_SIZE * blockIdx.y;

1086 auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;

1087

@@ -1108,10 +1133,10 @@ __global__ void reconstruct_exllama_2bit_kernel(

1133 int4 load_int4 = *b_ptr4;

1134

1135 half2 dq[4][8];

1111 - dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1);

1112 - dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1);

1113 - dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1);

1114 - dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1);

1136 + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + zero_offset);

1137 + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + zero_offset);

1138 + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + zero_offset);

1139 + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + zero_offset);

1140

1141 b_ptr += size_n;

1142 // half* dqh = (half*)dq;

@@ -1143,7 +1168,7 @@ void reconstruct_exllama(const uint32_t* b_q_weight,

1168 const uint32_t* b_gptq_qzeros,

1169 const half* b_gptq_scales, const int* b_q_perm,

1170 half* out, int height, int width, int groups,

1146 - int bit) {

1171 + bool use_v2_format, int bit) {

1172 dim3 blockDim, gridDim;

1173 blockDim.x = BLOCK_KN_SIZE;

1174 blockDim.y = 1;

@@ -1162,14 +1187,14 @@ void reconstruct_exllama(const uint32_t* b_q_weight,

1187 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();

1188 reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>(

1189 b_q_weight, b_q_perm, b_gptq_qzeros, b_gptq_scales, height, width, groups,

1165 - out);

1190 + use_v2_format, out);

1191 }

1192

1193 __global__ void gemm_half_q_half_alt_4bit_kernel(

1194 const half2* __restrict__ vec, const uint32_t* __restrict__ mat,

1195 half* __restrict__ mul, const half* __restrict__ scales,

1196 const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,

1172 - int batch, int height, int width) {

1197 + int batch, int height, int width, bool use_v2_format) {

1198 int zero_width = width / 8;

1199 int vec_height = height * 4;

1200 const int blockwidth2 = BLOCK_KN_SIZE / 2;

@@ -1179,6 +1204,9 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(

1204 int h_end = min(BLOCK_KN_SIZE / 8, height - h) * 4;

1205 auto w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;

1206

1207 + // GPTQv2 and GPTQv1 handles zero points differently

1208 + int zero_offset = use_v2_format ? 0 : 1;

1209 +

1210 __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];

1211 if (threadIdx.x < h_end) {

1212 for (int m = 0; m < b_end; ++m) {

@@ -1223,10 +1251,11 @@ __global__ void gemm_half_q_half_alt_4bit_kernel(

1251 half2 zero = __halves2half2(

1252 __hmul(scale_f,

1253 __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xF) -

1226 - 1)),

1227 - __hmul(scale_f2,

1228 - __int2half_rn(

1229 - -((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) - 1)));

1254 + zero_offset)),

1255 + __hmul(

1256 + scale_f2,

1257 + __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) -

1258 + zero_offset)));

1259 scales_tmp[tmp_k] = scale;

1260 zeros_tmp[tmp_k] = zero;

1261 }

@@ -1268,7 +1297,7 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(

1297 const half2* __restrict__ vec, const uint32_t* __restrict__ mat,

1298 half* __restrict__ mul, const half* __restrict__ scales,

1299 const uint32_t* __restrict__ zeros, const int* __restrict__ g_idx,

1271 - int batch, int height, int width) {

1300 + int batch, int height, int width, bool use_v2_format) {

1301 int zero_width = width / 4;

1302 int vec_height = height * 2;

1303 const int blockwidth2 = BLOCK_KN_SIZE / 2;

@@ -1278,6 +1307,9 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(

1307 int h_end = min(BLOCK_KN_SIZE / 4, height - h) * 2;

1308 auto w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;

1309

1310 + // GPTQv2 and GPTQv1 handles zero points differently

1311 + int zero_offset = use_v2_format ? 0 : 1;

1312 +

1313 __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2];

1314 if (threadIdx.x < h_end) {

1315 for (int m = 0; m < b_end; ++m) {

@@ -1312,12 +1344,13 @@ __global__ void gemm_half_q_half_alt_8bit_kernel(

1344 half scale_f2 = scales[g2 * width + w];

1345 half2 scale = __halves2half2(scale_f, scale_f2);

1346 half2 zero = __halves2half2(

1315 - __hmul(scale_f,

1316 - __int2half_rn(

1317 - -((zeros[g * zero_width + z_w] >> z_mod) & 0xff) - 1)),

1318 - __hmul(scale_f2,

1319 - __int2half_rn(

1320 - -((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) - 1)));

1347 + __hmul(scale_f, __int2half_rn(

1348 + -((zeros[g * zero_width + z_w] >> z_mod) & 0xff) -

1349 + zero_offset)),

1350 + __hmul(

1351 + scale_f2,

1352 + __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) -

1353 + zero_offset)));

1354 scales_tmp[tmp_k] = scale;

1355 zeros_tmp[tmp_k] = zero;

1356 }

@@ -1355,7 +1388,7 @@ void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,

1388 const uint32_t* b_gptq_qzeros,

1389 const half* b_gptq_scales, const int* b_g_idx,

1390 half* c, int size_m, int size_n, int size_k,

1358 - int bit) {

1391 + bool use_v2_format, int bit) {

1392 dim3 blockDim, gridDim;

1393 blockDim.x = BLOCK_KN_SIZE;

1394 blockDim.y = 1;

@@ -1372,17 +1405,15 @@ void gemm_half_q_half_alt(const half* a, const uint32_t* b_q_weight,

1405 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();

1406 kernel<<<gridDim, blockDim, 0, stream>>>(

1407 (const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, b_g_idx,

1375 - size_m, size_k / 32 * bit, size_n);

1408 + size_m, size_k / 32 * bit, size_n, use_v2_format);

1409 }

1410

1411 template <class T, int bit>

1379 -__global__ void reconstruct_gptq_kernel(const uint32_t* __restrict__ w,

1380 - const half* __restrict__ w_scales,

1381 - const uint32_t* __restrict__ w_zeros,

1382 - const int* __restrict__ g_idx,

1383 - const int height, const int width,

1384 - const int group,

1385 - half* __restrict__ out) {

1412 +__global__ void reconstruct_gptq_kernel(

1413 + const uint32_t* __restrict__ w, const half* __restrict__ w_scales,

1414 + const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,

1415 + const int height, const int width, const int group,

1416 + const bool use_v2_format, half* __restrict__ out) {

1417 // Start of block

1418

1419 auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;

@@ -1395,6 +1426,9 @@ __global__ void reconstruct_gptq_kernel(const uint32_t* __restrict__ w,

1426 MatrixView_half w_scales_(w_scales, group, width);

1427 T w_zeros_(w_zeros, group, width);

1428

1429 + // GPTQv2 and GPTQv1 handles zero points differently

1430 + int zero_offset = use_v2_format ? 0 : 1;

1431 +

1432 uint32_t w_read = w[blockIdx.y * width + column];

1433 half* out_ptr = out_.item_ptr(row, column);

1434

@@ -1402,7 +1436,7 @@ __global__ void reconstruct_gptq_kernel(const uint32_t* __restrict__ w,

1436 for (int s = 0; s < 32; s += bit) {

1437 int group = g_idx[row + s / bit];

1438 half w_scale = w_scales_.item(group, column);

1405 - uint32_t w_zero = w_zeros_.item(group, column) + 1;

1439 + uint32_t w_zero = w_zeros_.item(group, column) + zero_offset;

1440 half w_item =

1441 __hmul(__int2half_rn((int)((w_read >> s) & ((1 << bit) - 1)) - w_zero),

1442 w_scale);

@@ -1415,7 +1449,7 @@ __global__ void reconstruct_gptq_3bit_kernel(

1449 const uint32_t* __restrict__ w, const half* __restrict__ w_scales,

1450 const uint32_t* __restrict__ w_zeros, const int* __restrict__ g_idx,

1451 const int height, const int width, const int group,

1418 - half* __restrict__ out) {

1452 + const bool use_v2_format, half* __restrict__ out) {

1453 // Start of block

1454 auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x;

1455 auto row = blockIdx.y * 32;

@@ -1427,6 +1461,9 @@ __global__ void reconstruct_gptq_3bit_kernel(

1461 MatrixView_half w_scales_(w_scales, group, width);

1462 MatrixView_q3_row w_zeros_(w_zeros, group, width);

1463

1464 + // GPTQv2 and GPTQv1 handles zero points differently

1465 + int zero_offset = use_v2_format ? 0 : 1;

1466 +

1467 uint32_t w1 = w[(blockIdx.y * 3) * width + column];

1468 uint32_t w2 = w[(blockIdx.y * 3 + 1) * width + column];

1469 uint32_t w3 = w[(blockIdx.y * 3 + 2) * width + column];

@@ -1436,7 +1473,7 @@ __global__ void reconstruct_gptq_3bit_kernel(

1473 for (int i = 0; i < 32; i += 1) {

1474 int group = g_idx[row + i];

1475 half w_scale = w_scales_.item(group, column);

1439 - uint32_t w_zero = w_zeros_.item(group, column) + 1;

1476 + uint32_t w_zero = w_zeros_.item(group, column) + zero_offset;

1477 int w_item;

1478 if (i == 10) {

1479 w_item = (w1 >> 30) | ((w2 << 2) & 0x4);

@@ -1456,7 +1493,8 @@ __global__ void reconstruct_gptq_3bit_kernel(

1493

1494 void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,

1495 const half* b_gptq_scales, const int* b_g_idx, half* out,

1459 - int height, int width, int groups, int bit) {

1496 + int height, int width, int groups, bool use_v2_format,

1497 + int bit) {

1498 dim3 blockDim, gridDim;

1499 blockDim.x = BLOCK_KN_SIZE;

1500 blockDim.y = 1;

@@ -1476,7 +1514,7 @@ void reconstruct_gptq(const uint32_t* b_q_weight, const uint32_t* b_gptq_qzeros,

1514 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();

1515 kernel<<<gridDim, blockDim, 0, stream>>>(b_q_weight, b_gptq_scales,

1516 b_gptq_qzeros, b_g_idx, height,

1479 - width, groups, out);

1517 + width, groups, use_v2_format, out);

1518 }

1519

1520 void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,

@@ -1484,7 +1522,8 @@ void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,

1522 const uint32_t* b_gptq_qzeros,

1523 const half* b_gptq_scales, const int* b_g_idx,

1524 half* c, half* temp_dq, int size_m, int size_n,

1487 - int size_k, int groups, bool use_exllama, int bit) {

1525 + int size_k, int groups, bool use_exllama,

1526 + bool use_v2_format, int bit) {

1527 bool use_reconstruct;

1528 if (use_exllama) {

1529 use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) ||

@@ -1498,10 +1537,10 @@ void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,

1537 // Reconstruct FP16 matrix, then cuBLAS

1538 if (use_exllama) {

1539 reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,

1501 - temp_dq, size_k, size_n, groups, bit);

1540 + temp_dq, size_k, size_n, groups, use_v2_format, bit);

1541 } else {

1542 reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,

1504 - temp_dq, size_k, size_n, groups, bit);

1543 + temp_dq, size_k, size_n, groups, use_v2_format, bit);

1544 }

1545

1546 const half alpha = __float2half(1.0f);

@@ -1517,18 +1556,18 @@ void gemm_half_q_half_cuda(cublasHandle_t cublas_handle, const half* a,

1556 if (max_chunks) {

1557 gemm_half_q_half_cuda_part(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,

1558 b_g_idx, c, last_chunk, size_n, size_k,

1520 - BLOCK_M_SIZE_MAX, groups, bit);

1559 + BLOCK_M_SIZE_MAX, groups, use_v2_format, bit);

1560 }

1561

1562 if (last_chunk_size) {

1524 - gemm_half_q_half_cuda_part(a + last_chunk * size_k, b_q_weight,

1525 - b_gptq_qzeros, b_gptq_scales, b_g_idx,

1526 - c + last_chunk * size_n, last_chunk_size,

1527 - size_n, size_k, last_chunk_size, groups, bit);

1563 + gemm_half_q_half_cuda_part(

1564 + a + last_chunk * size_k, b_q_weight, b_gptq_qzeros, b_gptq_scales,

1565 + b_g_idx, c + last_chunk * size_n, last_chunk_size, size_n, size_k,

1566 + last_chunk_size, groups, use_v2_format, bit);

1567 }

1568 } else {

1569 gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx,

1531 - c, size_m, size_n, size_k, bit);

1570 + c, size_m, size_n, size_k, use_v2_format, bit);

1571 }

1572 }

1573

@@ -1815,7 +1854,7 @@ void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height,

1854 torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,

1855 torch::Tensor b_gptq_qzeros,

1856 torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,

1818 - bool use_exllama, int64_t bit) {

1857 + bool use_exllama, bool use_v2_format, int64_t bit) {

1858 const at::cuda::OptionalCUDAGuard device_guard(device_of(a));

1859 auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());

1860 at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options);

@@ -1833,7 +1872,7 @@ torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,

1872 c.size(1), // n

1873 a.size(1), // k

1874 b_gptq_qzeros.size(0), // group number

1836 - use_exllama, bit);

1875 + use_exllama, use_v2_format, bit);

1876 return c;

1877 }

1878

csrc/torch_bindings.cpp

+2 -1

@@ -557,7 +557,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {

557 // to prevent the meta function registry.

558 ops.def(

559 "gptq_gemm(Tensor a, Tensor b_q_weight, Tensor b_gptq_qzeros, "

560 - "Tensor b_gptq_scales, Tensor b_g_idx, bool use_exllama, int bit) "

560 + "Tensor b_gptq_scales, Tensor b_g_idx, bool use_exllama, bool "

561 + "use_v2_format, int bit) "

562 "-> Tensor",

563 {stride_tag});

564 ops.impl("gptq_gemm", torch::kCUDA, &gptq_gemm);

tests/kernels/quantization/test_gptq.py

+7 -1

@@ -26,4 +26,10 @@ def test_gptq_gemm_opcheck():

26 idx = torch.empty((0,), device="cuda", dtype=torch.int32)

27 use_exllama = True

28 bit = 4

29 - opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, bit))

29 + # Test both GPTQv1 and GPTQv2 format

30 + opcheck(

31 + torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, True, bit)

32 + )

33 + opcheck(

34 + torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, False, bit)

35 + )

tests/quantization/test_gptq_v2.py new

+109

@@ -0,0 +1,109 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Tests whether vllm correctly load and run gptq_v2 format checkpoints.

4 +

5 +Run `pytest tests/quantization/test_gptq_v2.py --forked`.

6 +"""

7 +

8 +import pytest

9 +import torch

10 +from transformers import AutoTokenizer

11 +

12 +from vllm import SamplingParams

13 +from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod

14 +

15 +# A dummy small model quantized by GPTQModel, stored in GPTQ v2 format

16 +MODELS = ["XXXXyu/Qwen3-1.7B-w2g64-gptq_v2"]

17 +

18 +# Generate multiple sequences for testing, because an 1.7B 2-bit model

19 +# cannot always generate normal texts.

20 +N_SEQ = 5

21 +

22 +

23 +@pytest.mark.parametrize("model_id", MODELS)

24 +def test_model_load(vllm_runner, model_id, monkeypatch):

25 + # `LLM.apply_model` requires pickling a function.

26 + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")

27 +

28 + # Only check the default GPTQ linear method (used for 2/3-bit models).

29 + # 4/8-bit linear methods like Marlin already support gptq_v2.

30 + linear_method_cls = GPTQLinearMethod

31 +

32 + with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm:

33 +

34 + def check_model(model_id):

35 + for name, submodule in model_id.named_modules():

36 + # Could check more modules if necessary

37 + if name == "model_id.layers.0.self_attn.qkv_proj":

38 + assert isinstance(submodule.quant_method, linear_method_cls)

39 +

40 + config = submodule.quant_method.quant_config

41 + assert config.checkpoint_format == "gptq_v2"

42 + assert submodule.quant_method.use_v2_format

43 +

44 + # Just break since currently we only check 1 module

45 + break

46 +

47 + # Check if gptq_v2 format is correctly loaded

48 + llm.apply_model(check_model)

49 +

50 +

51 +@pytest.mark.parametrize("model_id", MODELS)

52 +def test_model_inference(vllm_runner, model_id):

53 + # Prepare prompt to test the model's generation result.

54 + prompt = "What is the meaning of life?"

55 + messages = [

56 + {"role": "system", "content": "You are a helpful assistant."},

57 + {"role": "user", "content": prompt},

58 + ]

59 + tokenizer = AutoTokenizer.from_pretrained(model_id)

60 + text = tokenizer.apply_chat_template(

61 + messages,

62 + tokenize=False,

63 + add_generation_prompt=True,

64 + enable_thinking=False, # If thinking model, set it to false

65 + )

66 + sampling_params = SamplingParams(

67 + n=N_SEQ,

68 + max_tokens=128,

69 + temperature=0.7,

70 + top_p=0.8,

71 + top_k=20,

72 + min_p=0,

73 + presence_penalty=2,

74 + )

75 +

76 + with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm:

77 + # Generate a response to verify inference correctness

78 + output = llm.generate(text, sampling_params)

79 +

80 + # Make sure the output exists

81 + assert output

82 + assert output[0][1]

83 + assert len(output[0][1]) == N_SEQ

84 +

85 + def has_normal_char_distribution(texts, min_len):

86 + for text in texts:

87 + # Response too short

88 + if len(text) < min_len:

89 + return False

90 +

91 + # Basic ratio checks

92 + letters = sum(c.isalpha() for c in text)

93 + spaces = sum(c.isspace() for c in text)

94 + total = len(text)

95 +

96 + letter_ratio = letters / total

97 + space_ratio = spaces / total

98 +

99 + # At least 1 normal text should exist within output sequences

100 + # Normal text should be mostly letters with reasonable spacing

101 + # Some magic numbers, could be adjusted

102 + if 0.5 <= letter_ratio <= 0.9 and 0.01 <= space_ratio <= 0.3:

103 + return True

104 + # No sequence contains normal text, output might be broken

105 + return False

106 +

107 + # Apply some simple checks for giberish output

108 + # Print the output sequences if failed

109 + assert has_normal_char_distribution(output[0][1], 5), output[0][1]

vllm/_custom_ops.py

+10 -1

@@ -451,10 +451,18 @@ def gptq_gemm(

451 b_gptq_scales: torch.Tensor,

452 b_g_idx: torch.Tensor,

453 use_exllama: bool,

454 + use_v2_format: bool,

455 bit: int,

456 ) -> torch.Tensor:

457 return torch.ops._C.gptq_gemm(

457 - a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, use_exllama, bit

458 + a,

459 + b_q_weight,

460 + b_gptq_qzeros,

461 + b_gptq_scales,

462 + b_g_idx,

463 + use_exllama,

464 + use_v2_format,

465 + bit,

466 )

467

468

@@ -468,6 +476,7 @@ if hasattr(torch.ops._C, "gptq_gemm"):

476 b_gptq_scales: torch.Tensor,

477 b_g_idx: torch.Tensor,

478 use_exllama: bool,

479 + use_v2_format: bool,

480 bit: int,

481 ) -> torch.Tensor:

482 return torch.empty(

vllm/model_executor/layers/quantization/gptq.py

+29 -1

@@ -11,6 +11,7 @@ from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE

11 from torch.nn.parameter import Parameter

13 from vllm import _custom_ops as ops

14 +from vllm.logger import init_logger

15 from vllm.model_executor.layers.fused_moe.layer import FusedMoE

16 from vllm.model_executor.layers.linear import LinearMethodBase

17 from vllm.model_executor.layers.quantization.base_config import (

@@ -36,6 +37,8 @@ if TYPE_CHECKING:

37 else:

38 QuantizationMethods = str

40 +logger = init_logger(__name__)

41 +

43 class GPTQConfig(QuantizationConfig):

44 """Config class for GPTQ.

@@ -52,6 +55,7 @@ class GPTQConfig(QuantizationConfig):

55 dynamic: dict[str, dict[str, int | bool]],

56 autoround_version: str = "",

57 modules_in_block_to_quantize: list[str] | None = None,

58 + checkpoint_format: str = "",

59 ) -> None:

60 # GPTQModel use `dynamic` config property to allow per module

61 # quantization config so each module can be individually optimized.

@@ -89,12 +93,24 @@ class GPTQConfig(QuantizationConfig):

93 "Currently, only 2/3/4/8-bit weight quantization is "

94 f"supported for GPTQ, but got {self.weight_bits} bits."

95 )

96 + # Somehow gptq_gemm 4-bit is buggy, maybe fix it in the future.

97 + # For now, show a warning, since gptq_marlin will be used by default.

98 + if self.weight_bits == 4:

99 + logger.warning_once(

100 + "Currently, the 4-bit gptq_gemm kernel for GPTQ is buggy. "

101 + "Please switch to gptq_marlin or gptq_bitblas."

102 + )

103

104 self.modules_in_block_to_quantize = modules_in_block_to_quantize or []

105

106 # used to identify GPTQ model quantized by autoround

107 self.autoround_version = autoround_version

108

109 + # GPTQ v1 and v2 format deals with zero points differently.

110 + # Currently GPTQModel stores v1 format checkpoints by default,

111 + # but provides the option to set `format="gptq_v2"` in `QuantizeConfig`.

112 + self.checkpoint_format = checkpoint_format

113 +

114 def __repr__(self) -> str:

115 return (

116 f"GPTQConfig(weight_bits={self.weight_bits}, "

@@ -102,7 +118,8 @@ class GPTQConfig(QuantizationConfig):

118 f"desc_act={self.desc_act}), "

119 f"lm_head_quantized={self.lm_head_quantized}, "

120 f"dynamic={self.dynamic}, "

105 - f"modules_in_block_to_quantize={self.modules_in_block_to_quantize})"

121 + f"modules_in_block_to_quantize={self.modules_in_block_to_quantize}), "

122 + f"checkpoint_format={self.checkpoint_format})"

123 )

124

125 @classmethod

@@ -137,6 +154,9 @@ class GPTQConfig(QuantizationConfig):

154 modules_in_block_to_quantize = cls.get_from_keys_or(

155 config, ["modules_in_block_to_quantize"], default=None

156 )

157 + checkpoint_format = cls.get_from_keys_or(

158 + config, ["checkpoint_format"], default=""

159 + )

160 return cls(

161 weight_bits,

162 group_size,

@@ -145,6 +165,7 @@ class GPTQConfig(QuantizationConfig):

165 dynamic,

166 autoround_version,

167 modules_in_block_to_quantize,

168 + checkpoint_format,

169 )

170

171 def get_quant_method(

@@ -154,6 +175,7 @@ class GPTQConfig(QuantizationConfig):

175 # GPTQ MoE support: fall back to MoeWNA16 for broad compatibility

176 from .moe_wna16 import MoeWNA16Config

177

178 + # TODO: maybe update this for GPTQv2 format checkpoints

179 config = {

180 "quant_method": "gptq",

181 "bits": self.weight_bits,

@@ -210,6 +232,9 @@ class GPTQLinearMethod(LinearMethodBase):

232 def __init__(self, quant_config: GPTQConfig):

233 self.quant_config = quant_config

234

235 + # GPTQ v1 and v2 format deals with zero points differently

236 + self.use_v2_format = quant_config.checkpoint_format == "gptq_v2"

237 +

238 def create_weights(

239 self,

240 layer: torch.nn.Module,

@@ -351,6 +376,8 @@ class GPTQLinearMethod(LinearMethodBase):

376 out_shape = x.shape[:-1] + (layer.qweight.shape[-1],)

377 reshaped_x = x.reshape(-1, x.shape[-1])

378

379 + # GPTQ v1 and v2 format checkpoints deals with zero points differently,

380 + # and require different gemm kernels.

381 output = ops.gptq_gemm(

382 reshaped_x,

383 layer.qweight,

@@ -358,6 +385,7 @@ class GPTQLinearMethod(LinearMethodBase):

385 layer.scales,

386 layer.g_idx,

387 layer.exllama_state == ExllamaState.READY,

388 + self.use_v2_format,

389 self.quant_config.weight_bits,

390 )

391 if bias is not None:

vllm/model_executor/layers/quantization/kernels/mixed_precision/exllama.py

+6 -1

@@ -145,10 +145,15 @@ class ExllamaLinearKernel(MPLinearKernel):

145

146 w_q, w_s, w_zp, w_g_idx = self._get_weight_params(layer)

147

148 + # gptq_gemm supports GPTQv2 format by passing use_v2_format=True.

149 + # However, the MPLinearLayerConfig doesn't contain format info.

150 + # So hardcode GPTQv1 format here, to keep its behavior unchanged.

151 + use_v2_format = False

152 +

153 assert w_zp is not None, "Zero points are required by Exllama"

154 assert w_g_idx is not None, "Group index is required by Exllama"

155 output = ops.gptq_gemm(

151 - x_2d, w_q, w_zp, w_s, w_g_idx, True, c.weight_type.size_bits

156 + x_2d, w_q, w_zp, w_s, w_g_idx, True, use_v2_format, c.weight_type.size_bits

157 )

158

159 if bias is not None: