A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Support SqueezeLLM (#1326)
Co-authored-by: squeeze-ai-lab <squeezeailab.bair@gmail.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
chooper1 committed
Oct 22, 2023 at 03:14 UTC
1f24755bf802a2061bd46f3dd1191b7898f13f45
16 files changed
+378
-40
benchmarks/benchmark_latency.py
+1
-1
@@ -70,7 +70,7 @@ if __name__ == '__main__':
70
parser.add_argument('--tokenizer', type=str, default=None)
71
parser.add_argument('--quantization',
72
'-q',
73
- choices=['awq', None],
73
+ choices=['awq', 'squeezellm', None],
74
default=None)
75
parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
76
parser.add_argument('--input-len', type=int, default=32)
benchmarks/benchmark_throughput.py
+1
-1
@@ -201,7 +201,7 @@ if __name__ == "__main__":
201
parser.add_argument("--tokenizer", type=str, default=None)
202
parser.add_argument('--quantization',
203
'-q',
204
- choices=['awq', None],
204
+ choices=['awq', 'squeezellm', None],
205
default=None)
206
parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
207
parser.add_argument("--n",
csrc/quantization.cpp
+8
-4
@@ -7,9 +7,13 @@ torch::Tensor awq_gemm(
7
torch::Tensor _zeros,
8
int split_k_iters);
9
10
+void squeezellm_gemm(
11
+ torch::Tensor vec,
12
+ torch::Tensor mat,
13
+ torch::Tensor mul,
14
+ torch::Tensor lookup_table);
15
+
16
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
11
- m.def(
12
- "awq_gemm",
13
- &awq_gemm,
14
- "Quantized GEMM for AWQ");
17
+ m.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
18
+ m.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
19
}
csrc/quantization/squeezellm/quant_cuda_kernel.cu
new
+148
@@ -0,0 +1,148 @@
1
+#include <torch/all.h>
2
+#include <torch/python.h>
3
+#include <cuda.h>
4
+#include <cuda_runtime.h>
5
+#include <cuda_fp16.h>
6
+
7
+// half-tensor
8
+#include <c10/cuda/CUDAStream.h>
9
+#include <ATen/cuda/CUDATensorMethods.cuh>
10
+
11
+#define BLOCKWIDTH 128
12
+#define BLOCKHEIGHT4 16
13
+
14
+namespace vllm {
15
+namespace squeezellm {
16
+
17
+__device__ inline unsigned int as_unsigned(int i) {
18
+ return *reinterpret_cast<unsigned int*>(&i);
19
+}
20
+
21
+// 4-bit matvec kernel (LUT-based)
22
+__global__ void NUQ4MatMulKernel(
23
+ const half2* __restrict__ vec,
24
+ const int* __restrict__ mat,
25
+ half2* __restrict__ mul,
26
+ const __half* __restrict__ lookup_table,
27
+ int height,
28
+ int width,
29
+ int batch,
30
+ int vec_height
31
+) {
32
+
33
+ const int blockwidth2 = BLOCKWIDTH / 2;
34
+
35
+ int row = BLOCKHEIGHT4 * blockIdx.x;
36
+ int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
37
+
38
+ __shared__ half2 blockvec[blockwidth2];
39
+
40
+ __shared__ __half deq2[16][BLOCKWIDTH];
41
+ int off = threadIdx.x;
42
+ int column_offset = col * 16;
43
+ for (int val = 0; val < 16; val += 1) {
44
+ int lut_index = column_offset + val;
45
+ deq2[val][off] = lookup_table[lut_index];
46
+ }
47
+
48
+ __half res;
49
+ half2 res2;
50
+ half2 tmp2;
51
+
52
+ int i;
53
+ int k;
54
+
55
+ unsigned int tmp1;
56
+ unsigned int lut_index1, lut_index2;
57
+
58
+ for (int b = 0; b < batch; ++b){
59
+ i = width * row + col;
60
+ res = __int2half_rd(0);
61
+ k = 0;
62
+
63
+ __syncthreads();
64
+ if (threadIdx.x < blockwidth2)
65
+ blockvec[threadIdx.x] = vec[b * vec_height / 2 + (row / BLOCKHEIGHT4) * blockwidth2 + threadIdx.x];
66
+ __syncthreads();
67
+
68
+ while (k < blockwidth2) {
69
+ tmp1 = as_unsigned(mat[i]);
70
+
71
+ res2 = {};
72
+ tmp2 = {};
73
+
74
+ lut_index1 = tmp1 & 0xF;
75
+ lut_index2 = (tmp1 >> 4) & 0xF;
76
+ tmp2.x = deq2[lut_index1][off];
77
+ tmp2.y = deq2[lut_index2][off];
78
+ res2 = __hfma2(tmp2, blockvec[k + 0], res2);
79
+
80
+ lut_index1 = (tmp1 >> 8) & 0xF;
81
+ lut_index2 = (tmp1 >> 12) & 0xF;
82
+ tmp2.x = deq2[lut_index1][off];
83
+ tmp2.y = deq2[lut_index2][off];
84
+ res2 = __hfma2(tmp2, blockvec[k + 1], res2);
85
+
86
+ lut_index1 = (tmp1 >> 16) & 0xF;
87
+ lut_index2 = (tmp1 >> 20) & 0xF;
88
+ tmp2.x = deq2[lut_index1][off];
89
+ tmp2.y = deq2[lut_index2][off];
90
+ res2 = __hfma2(tmp2, blockvec[k + 2], res2);
91
+
92
+ lut_index1 = (tmp1 >> 24) & 0xF;
93
+ lut_index2 = (tmp1 >> 28) & 0xF;
94
+ tmp2.x = deq2[lut_index1][off];
95
+ tmp2.y = deq2[lut_index2][off];
96
+ res2 = __hfma2(tmp2, blockvec[k + 3], res2);
97
+
98
+ res = __hadd(__hadd(res2.x, res2.y), res);
99
+
100
+ i += width;
101
+ k += 4;
102
+ }
103
+
104
+ // col%2 -> only set one of the two values
105
+ half2 res3 = {};
106
+ if (col % 2 == 0) {
107
+ res3.x = res;
108
+ } else {
109
+ res3.y = res;
110
+ }
111
+
112
+ atomicAdd(&mul[b * width / 2 + col / 2], res3);
113
+ }
114
+}
115
+
116
+} // namespace squeezellm
117
+} // namespace vllm
118
+
119
+// 4-bit matvec kernel (LUT-based)
120
+void squeezellm_gemm(
121
+ torch::Tensor vec,
122
+ torch::Tensor mat,
123
+ torch::Tensor mul,
124
+ torch::Tensor lookup_table
125
+) {
126
+ int height = mat.size(0);
127
+ int width = mat.size(1);
128
+
129
+ int batch = vec.size(0);
130
+ int vec_height = vec.size(1);
131
+
132
+ dim3 blocks(
133
+ (height + BLOCKHEIGHT4 - 1) / BLOCKHEIGHT4,
134
+ (width + BLOCKWIDTH - 1) / BLOCKWIDTH
135
+ );
136
+ dim3 threads(BLOCKWIDTH);
137
+
138
+ vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads>>>(
139
+ (half2*) vec.data<at::Half>(),
140
+ mat.data_ptr<int>(),
141
+ (half2*) mul.data<at::Half>(),
142
+ (__half*) lookup_table.data<at::Half>(),
143
+ height, width, batch, vec_height
144
+ );
145
+}
146
+
147
+#undef BLOCKWIDTH
148
+#undef BLOCKHEIGHT4
setup.py
+1
@@ -200,6 +200,7 @@ quantization_extension = CUDAExtension(
200
sources=[
201
"csrc/quantization.cpp",
202
"csrc/quantization/awq/gemm_kernels.cu",
203
+ "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
204
],
205
extra_compile_args={
206
"cxx": CXX_FLAGS,
vllm/config.py
+1
-1
@@ -103,7 +103,7 @@ class ModelConfig:
103
self.tokenizer_mode = tokenizer_mode
104
105
def _verify_quantization(self) -> None:
106
- supported_quantization = ["awq"]
106
+ supported_quantization = ["awq", "squeezellm"]
107
if self.quantization is None:
108
return
109
quantization = self.quantization.lower()
vllm/engine/arg_utils.py
+1
-1
@@ -168,7 +168,7 @@ class EngineArgs:
168
parser.add_argument('--quantization',
169
'-q',
170
type=str,
171
- choices=['awq', None],
171
+ choices=['awq', 'squeezellm', None],
172
default=None,
173
help='Method used to quantize the weights')
174
return parser
vllm/model_executor/layers/quantized_linear/__init__.py
+4
@@ -1,10 +1,14 @@
1
from vllm.model_executor.layers.quantized_linear.awq import (
2
AWQColumnParallelLinear, AWQRowParallelLinear)
3
+from vllm.model_executor.layers.quantized_linear.squeezellm import (
4
+ SqueezeLLMColumnParallelLinear, SqueezeLLMRowParallelLinear)
5
from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
6
RowParallelLinear)
7
8
_QUANTIZED_LINEAR_REGISTRY = {
9
"awq": (AWQColumnParallelLinear, AWQRowParallelLinear),
10
+ "squeezellm":
11
+ (SqueezeLLMColumnParallelLinear, SqueezeLLMRowParallelLinear),
12
}
13
14
vllm/model_executor/layers/quantized_linear/awq.py
+9
-5
@@ -11,9 +11,11 @@ from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
11
class AWQColumnParallelLinear(ColumnParallelLinear):
12
13
def create_weights(self, dtype: torch.dtype) -> None:
14
- assert self.input_size % self.quant_config.weight_bits == 0
15
- assert (self.output_size_per_partition %
16
- self.quant_config.pack_factor == 0)
14
+ assert self.input_size % self.quant_config.group_size == 0
15
+ if self.output_size_per_partition % self.quant_config.pack_factor != 0:
16
+ raise ValueError(
17
+ "The tensor parallel size is not aligned with the quantized "
18
+ "weight shape. Please use a different tensor parallel size.")
19
self.qweight = Parameter(
20
torch.empty(
21
self.input_size,
@@ -62,9 +64,11 @@ class AWQColumnParallelLinear(ColumnParallelLinear):
64
class AWQRowParallelLinear(RowParallelLinear):
65
66
def create_weights(self, dtype: torch.dtype) -> None:
65
- assert (self.input_size_per_partition %
66
- self.quant_config.weight_bits == 0)
67
assert self.output_size % self.quant_config.pack_factor == 0
68
+ if self.input_size_per_partition % self.quant_config.group_size != 0:
69
+ raise ValueError(
70
+ "The tensor parallel size is not aligned with the quantized "
71
+ "weight shape. Please use a different tensor parallel size.")
72
self.qweight = Parameter(
73
torch.empty(
74
self.input_size_per_partition,
vllm/model_executor/layers/quantized_linear/squeezellm.py
new
+84
@@ -0,0 +1,84 @@
1
+from typing import Optional
2
+
3
+import torch
4
+from torch.nn.parameter import Parameter
5
+
6
+from vllm import quantization_ops
7
+from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
8
+ RowParallelLinear)
9
+
10
+
11
+class SqueezeLLMColumnParallelLinear(ColumnParallelLinear):
12
+
13
+ def create_weights(self, dtype: torch.dtype) -> None:
14
+ assert self.input_size % self.quant_config.pack_factor == 0
15
+ self.qweight = Parameter(
16
+ torch.empty(
17
+ self.input_size // self.quant_config.pack_factor,
18
+ self.output_size_per_partition,
19
+ device="cuda",
20
+ dtype=torch.int32,
21
+ ),
22
+ requires_grad=False,
23
+ )
24
+ self.lookup_table = Parameter(
25
+ torch.empty(
26
+ self.output_size_per_partition,
27
+ self.quant_config.weight_bits**2,
28
+ device="cuda",
29
+ dtype=dtype,
30
+ ),
31
+ requires_grad=False,
32
+ )
33
+
34
+ def apply_weights(
35
+ self,
36
+ x: torch.Tensor,
37
+ bias: Optional[torch.Tensor],
38
+ ) -> torch.Tensor:
39
+ out_shape = x.shape[:-1] + (self.qweight.shape[-1], )
40
+ reshaped_x = x.reshape(-1, x.shape[-1])
41
+ # NOTE: The output tensor should be zero-initialized.
42
+ out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
43
+ quantization_ops.squeezellm_gemm(reshaped_x, self.qweight, out,
44
+ self.lookup_table)
45
+
46
+ if bias is not None:
47
+ out = out + bias
48
+ return out.reshape(out_shape)
49
+
50
+
51
+class SqueezeLLMRowParallelLinear(RowParallelLinear):
52
+
53
+ def create_weights(self, dtype: torch.dtype) -> None:
54
+ if self.input_size_per_partition % self.quant_config.pack_factor != 0:
55
+ raise ValueError(
56
+ "The tensor parallel size is not aligned with the quantized "
57
+ "weight shape. Please use a different tensor parallel size.")
58
+ self.qweight = Parameter(
59
+ torch.empty(
60
+ self.input_size_per_partition // self.quant_config.pack_factor,
61
+ self.output_size,
62
+ device="cuda",
63
+ dtype=torch.int32,
64
+ ),
65
+ requires_grad=False,
66
+ )
67
+ self.lookup_table = Parameter(
68
+ torch.empty(
69
+ self.output_size,
70
+ self.quant_config.weight_bits**2,
71
+ device="cuda",
72
+ dtype=dtype,
73
+ ),
74
+ requires_grad=False,
75
+ )
76
+
77
+ def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
78
+ reshaped_x = x.reshape(-1, x.shape[-1])
79
+ out_shape = x.shape[:-1] + (self.qweight.shape[-1], )
80
+ # NOTE: The output tensor should be zero-initialized.
81
+ out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
82
+ quantization_ops.squeezellm_gemm(reshaped_x, self.qweight, out,
83
+ self.lookup_table)
84
+ return out.reshape(out_shape)
vllm/model_executor/models/llama.py
+15
-9
@@ -314,17 +314,21 @@ class LlamaForCausalLM(nn.Module):
314
load_format: str = "auto",
315
revision: Optional[str] = None):
316
if self.quant_config is None:
317
- weight_suffixes = ["weight"]
317
+ col_weight_suffixes = ["weight"]
318
+ row_weight_suffixes = ["weight"]
319
else:
319
- weight_suffixes = self.quant_config.get_tp_tensor_names()
320
+ col_weight_suffixes = (
321
+ self.quant_config.get_col_parallel_tensor_names())
322
+ row_weight_suffixes = (
323
+ self.quant_config.get_row_parallel_tensor_names())
324
325
column_parallel_weights: List[str] = []
326
for layer in self._column_parallel_layers:
323
- for suffix in weight_suffixes:
327
+ for suffix in col_weight_suffixes:
328
column_parallel_weights.append(f"{layer}.{suffix}")
329
row_parallel_weights: List[str] = []
330
for layer in self._row_parallel_layers:
327
- for suffix in weight_suffixes:
331
+ for suffix in row_weight_suffixes:
332
row_parallel_weights.append(f"{layer}.{suffix}")
333
334
tp_size = get_tensor_model_parallel_world_size()
@@ -351,10 +355,10 @@ class LlamaForCausalLM(nn.Module):
355
if "rotary_emb.inv_freq" in name:
356
continue
357
354
- is_packed = False
358
+ packed_dim = None
359
is_transposed = False
360
if self.quant_config is not None:
357
- is_packed = self.quant_config.is_packed(name)
361
+ packed_dim = self.quant_config.get_packed_dim(name)
362
is_transposed = self.quant_config.is_transposed(name)
363
if is_transposed:
364
loaded_weight = convert_pyslice_to_tensor(loaded_weight)
@@ -368,9 +372,11 @@ class LlamaForCausalLM(nn.Module):
372
if is_transposed:
373
param = param.T
374
371
- if is_packed:
372
- shard_size //= self.quant_config.pack_factor
373
- offset //= self.quant_config.pack_factor
375
+ if packed_dim is not None:
376
+ shard_dim = 0 if not is_transposed else 1
377
+ if packed_dim == shard_dim:
378
+ shard_size //= self.quant_config.pack_factor
379
+ offset //= self.quant_config.pack_factor
380
381
if weight_name in ["k_proj", "v_proj"]:
382
shard_id = tp_rank // num_kv_heads_replicas
vllm/model_executor/models/mistral.py
+15
-9
@@ -298,17 +298,21 @@ class MistralForCausalLM(nn.Module):
298
load_format: str = "auto",
299
revision: Optional[str] = None):
300
if self.quant_config is None:
301
- weight_suffixes = ["weight"]
301
+ col_weight_suffixes = ["weight"]
302
+ row_weight_suffixes = ["weight"]
303
else:
303
- weight_suffixes = self.quant_config.get_tp_tensor_names()
304
+ col_weight_suffixes = (
305
+ self.quant_config.get_col_parallel_tensor_names())
306
+ row_weight_suffixes = (
307
+ self.quant_config.get_row_parallel_tensor_names())
308
309
column_parallel_weights: List[str] = []
310
for layer in self._column_parallel_layers:
307
- for suffix in weight_suffixes:
311
+ for suffix in col_weight_suffixes:
312
column_parallel_weights.append(f"{layer}.{suffix}")
313
row_parallel_weights: List[str] = []
314
for layer in self._row_parallel_layers:
311
- for suffix in weight_suffixes:
315
+ for suffix in row_weight_suffixes:
316
row_parallel_weights.append(f"{layer}.{suffix}")
317
318
tp_size = get_tensor_model_parallel_world_size()
@@ -331,10 +335,10 @@ class MistralForCausalLM(nn.Module):
335
if "rotary_emb.inv_freq" in name:
336
continue
337
334
- is_packed = False
338
+ packed_dim = None
339
is_transposed = False
340
if self.quant_config is not None:
337
- is_packed = self.quant_config.is_packed(name)
341
+ packed_dim = self.quant_config.get_packed_dim(name)
342
is_transposed = self.quant_config.is_transposed(name)
343
if is_transposed:
344
loaded_weight = convert_pyslice_to_tensor(loaded_weight)
@@ -348,9 +352,11 @@ class MistralForCausalLM(nn.Module):
352
if is_transposed:
353
param = param.T
354
351
- if is_packed:
352
- shard_size //= self.quant_config.pack_factor
353
- offset //= self.quant_config.pack_factor
355
+ if packed_dim is not None:
356
+ shard_dim = 0 if not is_transposed else 1
357
+ if packed_dim == shard_dim:
358
+ shard_size //= self.quant_config.pack_factor
359
+ offset //= self.quant_config.pack_factor
360
361
loaded_weight = loaded_weight[
362
shard_size * tensor_model_parallel_rank:shard_size *
vllm/model_executor/quantization_utils/__init__.py
+2
@@ -2,9 +2,11 @@ from typing import Type
2
3
from vllm.model_executor.quantization_utils.awq import AWQConfig
4
from vllm.model_executor.quantization_utils.base import QuantizationConfig
5
+from vllm.model_executor.quantization_utils.squeezellm import SqueezeLLMConfig
6
7
_QUANTIZATION_REGISTRY = {
8
"awq": AWQConfig,
9
+ "squeezellm": SqueezeLLMConfig,
10
}
11
12
vllm/model_executor/quantization_utils/awq.py
+7
-3
@@ -60,13 +60,17 @@ class AWQConfig(QuantizationConfig):
60
return cls(weight_bits, group_size, zero_point)
61
62
@classmethod
63
- def get_packed_tensor_names(cls) -> List[str]:
64
- return ["qweight", "qzeros"]
63
+ def get_packed_tensors(cls) -> Dict[str, int]:
64
+ return {"qweight": 1, "qzeros": 1}
65
66
@classmethod
67
def get_transposed_tensor_names(cls) -> List[str]:
68
return ["qweight", "qzeros", "scales"]
69
70
@classmethod
71
- def get_tp_tensor_names(cls) -> List[str]:
71
+ def get_col_parallel_tensor_names(cls) -> List[str]:
72
+ return ["qweight", "qzeros", "scales"]
73
+
74
+ @classmethod
75
+ def get_row_parallel_tensor_names(cls) -> List[str]:
76
return ["qweight", "qzeros", "scales"]
vllm/model_executor/quantization_utils/base.py
+16
-6
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List
1
+from typing import Any, Dict, List, Optional
2
3
import torch
4
@@ -45,19 +45,25 @@ class QuantizationConfig:
45
"quantization config.")
46
47
@classmethod
48
- def get_packed_tensor_names(cls) -> List[str]:
48
+ def get_packed_tensors(cls) -> Dict[str, int]:
49
+ """Returns a dictionary of packed tensor names and their pack dims."""
50
raise NotImplementedError
51
52
@classmethod
52
- def is_packed(cls, tensor_name: str) -> bool:
53
- """Returns True if a tensor is packed.
53
+ def get_packed_dim(cls, tensor_name: str) -> Optional[int]:
54
+ """Returns the pack dim of a tensor if it is packed.
55
56
A tensor is considered packed if each element in the tensor is a
57
packed representation of multiple elements in the original tensor.
58
For example, an INT32 element in the tensor may represent 8 INT4
59
elements in the original tensor.
60
+ If the tensor is not packed, returns None.
61
"""
60
- return any(tag in tensor_name for tag in cls.get_packed_tensor_names())
62
+ packed_tensors = cls.get_packed_tensors()
63
+ for packed_tensor_name, pack_dim in packed_tensors.items():
64
+ if packed_tensor_name in tensor_name:
65
+ return pack_dim
66
+ return None
67
68
@classmethod
69
def get_transposed_tensor_names(cls) -> List[str]:
@@ -71,5 +77,9 @@ class QuantizationConfig:
77
for tag in cls.get_transposed_tensor_names())
78
79
@classmethod
74
- def get_tp_tensor_names(cls) -> List[str]:
80
+ def get_col_parallel_tensor_names(cls) -> List[str]:
81
+ raise NotImplementedError
82
+
83
+ @classmethod
84
+ def get_row_parallel_tensor_names(cls) -> List[str]:
85
raise NotImplementedError
vllm/model_executor/quantization_utils/squeezellm.py
new
+65
@@ -0,0 +1,65 @@
1
+from typing import Any, Dict, List
2
+
3
+import torch
4
+
5
+from vllm.model_executor.quantization_utils.base import QuantizationConfig
6
+
7
+
8
+class SqueezeLLMConfig(QuantizationConfig):
9
+ """Config class for SqueezeLLM.
10
+
11
+ Reference: https://arxiv.org/pdf/2306.07629
12
+ """
13
+
14
+ def __init__(
15
+ self,
16
+ weight_bits: int,
17
+ ) -> None:
18
+ self.weight_bits = weight_bits
19
+
20
+ if self.weight_bits != 4:
21
+ raise ValueError(
22
+ "Currently, only 4-bit weight quantization is supported for "
23
+ f"SqueezeLLM, but got {self.weight_bits} bits.")
24
+
25
+ self.pack_factor = 32 // self.weight_bits
26
+
27
+ def __repr__(self) -> str:
28
+ return f"SqueezeLLMConfig(weight_bits={self.weight_bits})"
29
+
30
+ @classmethod
31
+ def get_name(cls) -> str:
32
+ return "squeezellm"
33
+
34
+ @classmethod
35
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
36
+ return [torch.half]
37
+
38
+ @classmethod
39
+ def get_min_capability(cls) -> int:
40
+ return 70
41
+
42
+ @classmethod
43
+ def get_config_filenames(cls) -> List[str]:
44
+ return ["quant_config.json"]
45
+
46
+ @classmethod
47
+ def from_config(cls, config: Dict[str, Any]) -> "SqueezeLLMConfig":
48
+ weight_bits = cls.get_from_keys(config, ["wbits"])
49
+ return cls(weight_bits)
50
+
51
+ @classmethod
52
+ def get_packed_tensors(cls) -> Dict[str, int]:
53
+ return {"qweight": 0}
54
+
55
+ @classmethod
56
+ def get_transposed_tensor_names(cls) -> List[str]:
57
+ return ["qweight"]
58
+
59
+ @classmethod
60
+ def get_col_parallel_tensor_names(cls) -> List[str]:
61
+ return ["qweight", "lookup_table"]
62
+
63
+ @classmethod
64
+ def get_row_parallel_tensor_names(cls) -> List[str]:
65
+ return ["qweight"]