@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f24755bf8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Support SqueezeLLM (#1326)

Co-authored-by: squeeze-ai-lab <squeezeailab.bair@gmail.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

chooper1 committed Oct 22, 2023 at 03:14 UTC 1f24755bf802a2061bd46f3dd1191b7898f13f45
16 files changed +378 -40
benchmarks/benchmark_latency.py
+1 -1
@@ -70,7 +70,7 @@ if __name__ == '__main__':
70 parser.add_argument('--tokenizer', type=str, default=None)
71 parser.add_argument('--quantization',
72 '-q',
73 - choices=['awq', None],
73 + choices=['awq', 'squeezellm', None],
74 default=None)
75 parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
76 parser.add_argument('--input-len', type=int, default=32)
benchmarks/benchmark_throughput.py
+1 -1
@@ -201,7 +201,7 @@ if __name__ == "__main__":
201 parser.add_argument("--tokenizer", type=str, default=None)
202 parser.add_argument('--quantization',
203 '-q',
204 - choices=['awq', None],
204 + choices=['awq', 'squeezellm', None],
205 default=None)
206 parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
207 parser.add_argument("--n",
csrc/quantization.cpp
+8 -4
@@ -7,9 +7,13 @@ torch::Tensor awq_gemm(
7 torch::Tensor _zeros,
8 int split_k_iters);
9
10 +void squeezellm_gemm(
11 + torch::Tensor vec,
12 + torch::Tensor mat,
13 + torch::Tensor mul,
14 + torch::Tensor lookup_table);
15 +
16 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
11 - m.def(
12 - "awq_gemm",
13 - &awq_gemm,
14 - "Quantized GEMM for AWQ");
17 + m.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
18 + m.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
19 }
csrc/quantization/squeezellm/quant_cuda_kernel.cu new
+148
@@ -0,0 +1,148 @@
1 +#include <torch/all.h>
2 +#include <torch/python.h>
3 +#include <cuda.h>
4 +#include <cuda_runtime.h>
5 +#include <cuda_fp16.h>
6 +
7 +// half-tensor
8 +#include <c10/cuda/CUDAStream.h>
9 +#include <ATen/cuda/CUDATensorMethods.cuh>
10 +
11 +#define BLOCKWIDTH 128
12 +#define BLOCKHEIGHT4 16
13 +
14 +namespace vllm {
15 +namespace squeezellm {
16 +
17 +__device__ inline unsigned int as_unsigned(int i) {
18 + return *reinterpret_cast<unsigned int*>(&i);
19 +}
20 +
21 +// 4-bit matvec kernel (LUT-based)
22 +__global__ void NUQ4MatMulKernel(
23 + const half2* __restrict__ vec,
24 + const int* __restrict__ mat,
25 + half2* __restrict__ mul,
26 + const __half* __restrict__ lookup_table,
27 + int height,
28 + int width,
29 + int batch,
30 + int vec_height
31 +) {
32 +
33 + const int blockwidth2 = BLOCKWIDTH / 2;
34 +
35 + int row = BLOCKHEIGHT4 * blockIdx.x;
36 + int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
37 +
38 + __shared__ half2 blockvec[blockwidth2];
39 +
40 + __shared__ __half deq2[16][BLOCKWIDTH];
41 + int off = threadIdx.x;
42 + int column_offset = col * 16;
43 + for (int val = 0; val < 16; val += 1) {
44 + int lut_index = column_offset + val;
45 + deq2[val][off] = lookup_table[lut_index];
46 + }
47 +
48 + __half res;
49 + half2 res2;
50 + half2 tmp2;
51 +
52 + int i;
53 + int k;
54 +
55 + unsigned int tmp1;
56 + unsigned int lut_index1, lut_index2;
57 +
58 + for (int b = 0; b < batch; ++b){
59 + i = width * row + col;
60 + res = __int2half_rd(0);
61 + k = 0;
62 +
63 + __syncthreads();
64 + if (threadIdx.x < blockwidth2)
65 + blockvec[threadIdx.x] = vec[b * vec_height / 2 + (row / BLOCKHEIGHT4) * blockwidth2 + threadIdx.x];
66 + __syncthreads();
67 +
68 + while (k < blockwidth2) {
69 + tmp1 = as_unsigned(mat[i]);
70 +
71 + res2 = {};
72 + tmp2 = {};
73 +
74 + lut_index1 = tmp1 & 0xF;
75 + lut_index2 = (tmp1 >> 4) & 0xF;
76 + tmp2.x = deq2[lut_index1][off];
77 + tmp2.y = deq2[lut_index2][off];
78 + res2 = __hfma2(tmp2, blockvec[k + 0], res2);
79 +
80 + lut_index1 = (tmp1 >> 8) & 0xF;
81 + lut_index2 = (tmp1 >> 12) & 0xF;
82 + tmp2.x = deq2[lut_index1][off];
83 + tmp2.y = deq2[lut_index2][off];
84 + res2 = __hfma2(tmp2, blockvec[k + 1], res2);
85 +
86 + lut_index1 = (tmp1 >> 16) & 0xF;
87 + lut_index2 = (tmp1 >> 20) & 0xF;
88 + tmp2.x = deq2[lut_index1][off];
89 + tmp2.y = deq2[lut_index2][off];
90 + res2 = __hfma2(tmp2, blockvec[k + 2], res2);
91 +
92 + lut_index1 = (tmp1 >> 24) & 0xF;
93 + lut_index2 = (tmp1 >> 28) & 0xF;
94 + tmp2.x = deq2[lut_index1][off];
95 + tmp2.y = deq2[lut_index2][off];
96 + res2 = __hfma2(tmp2, blockvec[k + 3], res2);
97 +
98 + res = __hadd(__hadd(res2.x, res2.y), res);
99 +
100 + i += width;
101 + k += 4;
102 + }
103 +
104 + // col%2 -> only set one of the two values
105 + half2 res3 = {};
106 + if (col % 2 == 0) {
107 + res3.x = res;
108 + } else {
109 + res3.y = res;
110 + }
111 +
112 + atomicAdd(&mul[b * width / 2 + col / 2], res3);
113 + }
114 +}
115 +
116 +} // namespace squeezellm
117 +} // namespace vllm
118 +
119 +// 4-bit matvec kernel (LUT-based)
120 +void squeezellm_gemm(
121 + torch::Tensor vec,
122 + torch::Tensor mat,
123 + torch::Tensor mul,
124 + torch::Tensor lookup_table
125 +) {
126 + int height = mat.size(0);
127 + int width = mat.size(1);
128 +
129 + int batch = vec.size(0);
130 + int vec_height = vec.size(1);
131 +
132 + dim3 blocks(
133 + (height + BLOCKHEIGHT4 - 1) / BLOCKHEIGHT4,
134 + (width + BLOCKWIDTH - 1) / BLOCKWIDTH
135 + );
136 + dim3 threads(BLOCKWIDTH);
137 +
138 + vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads>>>(
139 + (half2*) vec.data<at::Half>(),
140 + mat.data_ptr<int>(),
141 + (half2*) mul.data<at::Half>(),
142 + (__half*) lookup_table.data<at::Half>(),
143 + height, width, batch, vec_height
144 + );
145 +}
146 +
147 +#undef BLOCKWIDTH
148 +#undef BLOCKHEIGHT4
setup.py
+1
@@ -200,6 +200,7 @@ quantization_extension = CUDAExtension(
200 sources=[
201 "csrc/quantization.cpp",
202 "csrc/quantization/awq/gemm_kernels.cu",
203 + "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
204 ],
205 extra_compile_args={
206 "cxx": CXX_FLAGS,
vllm/config.py
+1 -1
@@ -103,7 +103,7 @@ class ModelConfig:
103 self.tokenizer_mode = tokenizer_mode
104
105 def _verify_quantization(self) -> None:
106 - supported_quantization = ["awq"]
106 + supported_quantization = ["awq", "squeezellm"]
107 if self.quantization is None:
108 return
109 quantization = self.quantization.lower()
vllm/engine/arg_utils.py
+1 -1
@@ -168,7 +168,7 @@ class EngineArgs:
168 parser.add_argument('--quantization',
169 '-q',
170 type=str,
171 - choices=['awq', None],
171 + choices=['awq', 'squeezellm', None],
172 default=None,
173 help='Method used to quantize the weights')
174 return parser
vllm/model_executor/layers/quantized_linear/__init__.py
+4
@@ -1,10 +1,14 @@
1 from vllm.model_executor.layers.quantized_linear.awq import (
2 AWQColumnParallelLinear, AWQRowParallelLinear)
3 +from vllm.model_executor.layers.quantized_linear.squeezellm import (
4 + SqueezeLLMColumnParallelLinear, SqueezeLLMRowParallelLinear)
5 from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
6 RowParallelLinear)
7
8 _QUANTIZED_LINEAR_REGISTRY = {
9 "awq": (AWQColumnParallelLinear, AWQRowParallelLinear),
10 + "squeezellm":
11 + (SqueezeLLMColumnParallelLinear, SqueezeLLMRowParallelLinear),
12 }
13
14
vllm/model_executor/layers/quantized_linear/awq.py
+9 -5
@@ -11,9 +11,11 @@ from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
11 class AWQColumnParallelLinear(ColumnParallelLinear):
12
13 def create_weights(self, dtype: torch.dtype) -> None:
14 - assert self.input_size % self.quant_config.weight_bits == 0
15 - assert (self.output_size_per_partition %
16 - self.quant_config.pack_factor == 0)
14 + assert self.input_size % self.quant_config.group_size == 0
15 + if self.output_size_per_partition % self.quant_config.pack_factor != 0:
16 + raise ValueError(
17 + "The tensor parallel size is not aligned with the quantized "
18 + "weight shape. Please use a different tensor parallel size.")
19 self.qweight = Parameter(
20 torch.empty(
21 self.input_size,
@@ -62,9 +64,11 @@ class AWQColumnParallelLinear(ColumnParallelLinear):
64 class AWQRowParallelLinear(RowParallelLinear):
65
66 def create_weights(self, dtype: torch.dtype) -> None:
65 - assert (self.input_size_per_partition %
66 - self.quant_config.weight_bits == 0)
67 assert self.output_size % self.quant_config.pack_factor == 0
68 + if self.input_size_per_partition % self.quant_config.group_size != 0:
69 + raise ValueError(
70 + "The tensor parallel size is not aligned with the quantized "
71 + "weight shape. Please use a different tensor parallel size.")
72 self.qweight = Parameter(
73 torch.empty(
74 self.input_size_per_partition,
vllm/model_executor/layers/quantized_linear/squeezellm.py new
+84
@@ -0,0 +1,84 @@
1 +from typing import Optional
2 +
3 +import torch
4 +from torch.nn.parameter import Parameter
5 +
6 +from vllm import quantization_ops
7 +from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
8 + RowParallelLinear)
9 +
10 +
11 +class SqueezeLLMColumnParallelLinear(ColumnParallelLinear):
12 +
13 + def create_weights(self, dtype: torch.dtype) -> None:
14 + assert self.input_size % self.quant_config.pack_factor == 0
15 + self.qweight = Parameter(
16 + torch.empty(
17 + self.input_size // self.quant_config.pack_factor,
18 + self.output_size_per_partition,
19 + device="cuda",
20 + dtype=torch.int32,
21 + ),
22 + requires_grad=False,
23 + )
24 + self.lookup_table = Parameter(
25 + torch.empty(
26 + self.output_size_per_partition,
27 + self.quant_config.weight_bits**2,
28 + device="cuda",
29 + dtype=dtype,
30 + ),
31 + requires_grad=False,
32 + )
33 +
34 + def apply_weights(
35 + self,
36 + x: torch.Tensor,
37 + bias: Optional[torch.Tensor],
38 + ) -> torch.Tensor:
39 + out_shape = x.shape[:-1] + (self.qweight.shape[-1], )
40 + reshaped_x = x.reshape(-1, x.shape[-1])
41 + # NOTE: The output tensor should be zero-initialized.
42 + out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
43 + quantization_ops.squeezellm_gemm(reshaped_x, self.qweight, out,
44 + self.lookup_table)
45 +
46 + if bias is not None:
47 + out = out + bias
48 + return out.reshape(out_shape)
49 +
50 +
51 +class SqueezeLLMRowParallelLinear(RowParallelLinear):
52 +
53 + def create_weights(self, dtype: torch.dtype) -> None:
54 + if self.input_size_per_partition % self.quant_config.pack_factor != 0:
55 + raise ValueError(
56 + "The tensor parallel size is not aligned with the quantized "
57 + "weight shape. Please use a different tensor parallel size.")
58 + self.qweight = Parameter(
59 + torch.empty(
60 + self.input_size_per_partition // self.quant_config.pack_factor,
61 + self.output_size,
62 + device="cuda",
63 + dtype=torch.int32,
64 + ),
65 + requires_grad=False,
66 + )
67 + self.lookup_table = Parameter(
68 + torch.empty(
69 + self.output_size,
70 + self.quant_config.weight_bits**2,
71 + device="cuda",
72 + dtype=dtype,
73 + ),
74 + requires_grad=False,
75 + )
76 +
77 + def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
78 + reshaped_x = x.reshape(-1, x.shape[-1])
79 + out_shape = x.shape[:-1] + (self.qweight.shape[-1], )
80 + # NOTE: The output tensor should be zero-initialized.
81 + out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
82 + quantization_ops.squeezellm_gemm(reshaped_x, self.qweight, out,
83 + self.lookup_table)
84 + return out.reshape(out_shape)
vllm/model_executor/models/llama.py
+15 -9
@@ -314,17 +314,21 @@ class LlamaForCausalLM(nn.Module):
314 load_format: str = "auto",
315 revision: Optional[str] = None):
316 if self.quant_config is None:
317 - weight_suffixes = ["weight"]
317 + col_weight_suffixes = ["weight"]
318 + row_weight_suffixes = ["weight"]
319 else:
319 - weight_suffixes = self.quant_config.get_tp_tensor_names()
320 + col_weight_suffixes = (
321 + self.quant_config.get_col_parallel_tensor_names())
322 + row_weight_suffixes = (
323 + self.quant_config.get_row_parallel_tensor_names())
324
325 column_parallel_weights: List[str] = []
326 for layer in self._column_parallel_layers:
323 - for suffix in weight_suffixes:
327 + for suffix in col_weight_suffixes:
328 column_parallel_weights.append(f"{layer}.{suffix}")
329 row_parallel_weights: List[str] = []
330 for layer in self._row_parallel_layers:
327 - for suffix in weight_suffixes:
331 + for suffix in row_weight_suffixes:
332 row_parallel_weights.append(f"{layer}.{suffix}")
333
334 tp_size = get_tensor_model_parallel_world_size()
@@ -351,10 +355,10 @@ class LlamaForCausalLM(nn.Module):
355 if "rotary_emb.inv_freq" in name:
356 continue
357
354 - is_packed = False
358 + packed_dim = None
359 is_transposed = False
360 if self.quant_config is not None:
357 - is_packed = self.quant_config.is_packed(name)
361 + packed_dim = self.quant_config.get_packed_dim(name)
362 is_transposed = self.quant_config.is_transposed(name)
363 if is_transposed:
364 loaded_weight = convert_pyslice_to_tensor(loaded_weight)
@@ -368,9 +372,11 @@ class LlamaForCausalLM(nn.Module):
372 if is_transposed:
373 param = param.T
374
371 - if is_packed:
372 - shard_size //= self.quant_config.pack_factor
373 - offset //= self.quant_config.pack_factor
375 + if packed_dim is not None:
376 + shard_dim = 0 if not is_transposed else 1
377 + if packed_dim == shard_dim:
378 + shard_size //= self.quant_config.pack_factor
379 + offset //= self.quant_config.pack_factor
380
381 if weight_name in ["k_proj", "v_proj"]:
382 shard_id = tp_rank // num_kv_heads_replicas
vllm/model_executor/models/mistral.py
+15 -9
@@ -298,17 +298,21 @@ class MistralForCausalLM(nn.Module):
298 load_format: str = "auto",
299 revision: Optional[str] = None):
300 if self.quant_config is None:
301 - weight_suffixes = ["weight"]
301 + col_weight_suffixes = ["weight"]
302 + row_weight_suffixes = ["weight"]
303 else:
303 - weight_suffixes = self.quant_config.get_tp_tensor_names()
304 + col_weight_suffixes = (
305 + self.quant_config.get_col_parallel_tensor_names())
306 + row_weight_suffixes = (
307 + self.quant_config.get_row_parallel_tensor_names())
308
309 column_parallel_weights: List[str] = []
310 for layer in self._column_parallel_layers:
307 - for suffix in weight_suffixes:
311 + for suffix in col_weight_suffixes:
312 column_parallel_weights.append(f"{layer}.{suffix}")
313 row_parallel_weights: List[str] = []
314 for layer in self._row_parallel_layers:
311 - for suffix in weight_suffixes:
315 + for suffix in row_weight_suffixes:
316 row_parallel_weights.append(f"{layer}.{suffix}")
317
318 tp_size = get_tensor_model_parallel_world_size()
@@ -331,10 +335,10 @@ class MistralForCausalLM(nn.Module):
335 if "rotary_emb.inv_freq" in name:
336 continue
337
334 - is_packed = False
338 + packed_dim = None
339 is_transposed = False
340 if self.quant_config is not None:
337 - is_packed = self.quant_config.is_packed(name)
341 + packed_dim = self.quant_config.get_packed_dim(name)
342 is_transposed = self.quant_config.is_transposed(name)
343 if is_transposed:
344 loaded_weight = convert_pyslice_to_tensor(loaded_weight)
@@ -348,9 +352,11 @@ class MistralForCausalLM(nn.Module):
352 if is_transposed:
353 param = param.T
354
351 - if is_packed:
352 - shard_size //= self.quant_config.pack_factor
353 - offset //= self.quant_config.pack_factor
355 + if packed_dim is not None:
356 + shard_dim = 0 if not is_transposed else 1
357 + if packed_dim == shard_dim:
358 + shard_size //= self.quant_config.pack_factor
359 + offset //= self.quant_config.pack_factor
360
361 loaded_weight = loaded_weight[
362 shard_size * tensor_model_parallel_rank:shard_size *
vllm/model_executor/quantization_utils/__init__.py
+2
@@ -2,9 +2,11 @@ from typing import Type
2
3 from vllm.model_executor.quantization_utils.awq import AWQConfig
4 from vllm.model_executor.quantization_utils.base import QuantizationConfig
5 +from vllm.model_executor.quantization_utils.squeezellm import SqueezeLLMConfig
6
7 _QUANTIZATION_REGISTRY = {
8 "awq": AWQConfig,
9 + "squeezellm": SqueezeLLMConfig,
10 }
11
12
vllm/model_executor/quantization_utils/awq.py
+7 -3
@@ -60,13 +60,17 @@ class AWQConfig(QuantizationConfig):
60 return cls(weight_bits, group_size, zero_point)
61
62 @classmethod
63 - def get_packed_tensor_names(cls) -> List[str]:
64 - return ["qweight", "qzeros"]
63 + def get_packed_tensors(cls) -> Dict[str, int]:
64 + return {"qweight": 1, "qzeros": 1}
65
66 @classmethod
67 def get_transposed_tensor_names(cls) -> List[str]:
68 return ["qweight", "qzeros", "scales"]
69
70 @classmethod
71 - def get_tp_tensor_names(cls) -> List[str]:
71 + def get_col_parallel_tensor_names(cls) -> List[str]:
72 + return ["qweight", "qzeros", "scales"]
73 +
74 + @classmethod
75 + def get_row_parallel_tensor_names(cls) -> List[str]:
76 return ["qweight", "qzeros", "scales"]
vllm/model_executor/quantization_utils/base.py
+16 -6
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List
1 +from typing import Any, Dict, List, Optional
2
3 import torch
4
@@ -45,19 +45,25 @@ class QuantizationConfig:
45 "quantization config.")
46
47 @classmethod
48 - def get_packed_tensor_names(cls) -> List[str]:
48 + def get_packed_tensors(cls) -> Dict[str, int]:
49 + """Returns a dictionary of packed tensor names and their pack dims."""
50 raise NotImplementedError
51
52 @classmethod
52 - def is_packed(cls, tensor_name: str) -> bool:
53 - """Returns True if a tensor is packed.
53 + def get_packed_dim(cls, tensor_name: str) -> Optional[int]:
54 + """Returns the pack dim of a tensor if it is packed.
55
56 A tensor is considered packed if each element in the tensor is a
57 packed representation of multiple elements in the original tensor.
58 For example, an INT32 element in the tensor may represent 8 INT4
59 elements in the original tensor.
60 + If the tensor is not packed, returns None.
61 """
60 - return any(tag in tensor_name for tag in cls.get_packed_tensor_names())
62 + packed_tensors = cls.get_packed_tensors()
63 + for packed_tensor_name, pack_dim in packed_tensors.items():
64 + if packed_tensor_name in tensor_name:
65 + return pack_dim
66 + return None
67
68 @classmethod
69 def get_transposed_tensor_names(cls) -> List[str]:
@@ -71,5 +77,9 @@ class QuantizationConfig:
77 for tag in cls.get_transposed_tensor_names())
78
79 @classmethod
74 - def get_tp_tensor_names(cls) -> List[str]:
80 + def get_col_parallel_tensor_names(cls) -> List[str]:
81 + raise NotImplementedError
82 +
83 + @classmethod
84 + def get_row_parallel_tensor_names(cls) -> List[str]:
85 raise NotImplementedError
vllm/model_executor/quantization_utils/squeezellm.py new
+65
@@ -0,0 +1,65 @@
1 +from typing import Any, Dict, List
2 +
3 +import torch
4 +
5 +from vllm.model_executor.quantization_utils.base import QuantizationConfig
6 +
7 +
8 +class SqueezeLLMConfig(QuantizationConfig):
9 + """Config class for SqueezeLLM.
10 +
11 + Reference: https://arxiv.org/pdf/2306.07629
12 + """
13 +
14 + def __init__(
15 + self,
16 + weight_bits: int,
17 + ) -> None:
18 + self.weight_bits = weight_bits
19 +
20 + if self.weight_bits != 4:
21 + raise ValueError(
22 + "Currently, only 4-bit weight quantization is supported for "
23 + f"SqueezeLLM, but got {self.weight_bits} bits.")
24 +
25 + self.pack_factor = 32 // self.weight_bits
26 +
27 + def __repr__(self) -> str:
28 + return f"SqueezeLLMConfig(weight_bits={self.weight_bits})"
29 +
30 + @classmethod
31 + def get_name(cls) -> str:
32 + return "squeezellm"
33 +
34 + @classmethod
35 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
36 + return [torch.half]
37 +
38 + @classmethod
39 + def get_min_capability(cls) -> int:
40 + return 70
41 +
42 + @classmethod
43 + def get_config_filenames(cls) -> List[str]:
44 + return ["quant_config.json"]
45 +
46 + @classmethod
47 + def from_config(cls, config: Dict[str, Any]) -> "SqueezeLLMConfig":
48 + weight_bits = cls.get_from_keys(config, ["wbits"])
49 + return cls(weight_bits)
50 +
51 + @classmethod
52 + def get_packed_tensors(cls) -> Dict[str, int]:
53 + return {"qweight": 0}
54 +
55 + @classmethod
56 + def get_transposed_tensor_names(cls) -> List[str]:
57 + return ["qweight"]
58 +
59 + @classmethod
60 + def get_col_parallel_tensor_names(cls) -> List[str]:
61 + return ["qweight", "lookup_table"]
62 +
63 + @classmethod
64 + def get_row_parallel_tensor_names(cls) -> List[str]:
65 + return ["qweight"]