A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Remove `SqueezeLLM` (#8220)
Dipika Sikka committed
Sep 6, 2024 at 18:29 UTC
23f322297f33a50dd1fe0870665d0c4414fd78ab
12 files changed
+6
-389
CMakeLists.txt
-1
@@ -181,7 +181,6 @@ set(VLLM_EXT_SRC
181
"csrc/pos_encoding_kernels.cu"
182
"csrc/activation_kernels.cu"
183
"csrc/layernorm_kernels.cu"
184
- "csrc/quantization/squeezellm/quant_cuda_kernel.cu"
184
"csrc/quantization/gptq/q_gemm.cu"
185
"csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
186
"csrc/quantization/fp8/common.cu"
csrc/ops.h
-3
@@ -170,9 +170,6 @@ void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
170
void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
171
torch::Tensor& scales);
172
173
-void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
174
- torch::Tensor lookup_table);
175
-
173
torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
174
torch::Tensor b_gptq_qzeros,
175
torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
csrc/quantization/squeezellm/quant_cuda_kernel.cu
deleted
-216
@@ -1,216 +0,0 @@
1
-#include <torch/all.h>
2
-#include <cuda.h>
3
-#include <cuda_runtime.h>
4
-#include <cuda_fp16.h>
5
-
6
-// half-tensor
7
-#include <c10/cuda/CUDAStream.h>
8
-#include <ATen/cuda/CUDATensorMethods.cuh>
9
-#include <c10/cuda/CUDAGuard.h>
10
-
11
-#define BLOCKWIDTH 128
12
-#define BLOCKHEIGHT4 16
13
-
14
-namespace vllm {
15
-namespace squeezellm {
16
-
17
-__device__ inline unsigned int as_unsigned(int i) {
18
- return *reinterpret_cast<unsigned int*>(&i);
19
-}
20
-
21
-// 4-bit matvec kernel (LUT-based)
22
-__global__ void NUQ4MatMulKernel(
23
-#ifndef USE_ROCM
24
- const half2* __restrict__ vec,
25
-#else
26
- const __half2* __restrict__ vec,
27
-#endif
28
- const int* __restrict__ mat,
29
-#ifndef USE_ROCM
30
- half2* __restrict__ mul,
31
-#else
32
- float2* __restrict__ mul,
33
-#endif
34
- const __half* __restrict__ lookup_table, int height, int width, int batch,
35
- int vec_height) {
36
-
37
- const int blockwidth2 = BLOCKWIDTH / 2;
38
-
39
- int row = BLOCKHEIGHT4 * blockIdx.x;
40
- int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
41
-
42
-#ifndef USE_ROCM
43
- __shared__ half2 blockvec[blockwidth2];
44
-#else
45
- __shared__ __half2 blockvec[blockwidth2];
46
-#endif
47
-
48
- __shared__ __half deq2[16][BLOCKWIDTH];
49
- int off = threadIdx.x;
50
- int column_offset = col * 16;
51
- for (int val = 0; val < 16; val += 1) {
52
- int lut_index = column_offset + val;
53
- deq2[val][off] = lookup_table[lut_index];
54
- }
55
-
56
- __half res;
57
-#ifndef USE_ROCM
58
- half2 res2;
59
- half2 tmp2;
60
-#else
61
- __half2 res2;
62
- __half2 tmp2;
63
-#endif
64
-
65
- int i;
66
- int k;
67
-
68
- unsigned int tmp1;
69
- unsigned int lut_index1, lut_index2;
70
-
71
- for (int b = 0; b < batch; ++b) {
72
- i = width * row + col;
73
- res = __int2half_rd(0);
74
- k = 0;
75
-
76
- __syncthreads();
77
- if (threadIdx.x < blockwidth2)
78
- blockvec[threadIdx.x] =
79
- vec[b * vec_height / 2 + (row / BLOCKHEIGHT4) * blockwidth2 +
80
- threadIdx.x];
81
- __syncthreads();
82
-
83
- while (k < blockwidth2) {
84
- tmp1 = as_unsigned(mat[i]);
85
-
86
-#ifndef USE_ROCM
87
- res2 = {};
88
- tmp2 = {};
89
-#else
90
- res2.x = __half_as_ushort(__float2half(0));
91
- res2.y = __half_as_ushort(__float2half(0));
92
- tmp2.x = __half_as_ushort(__float2half(0));
93
- tmp2.y = __half_as_ushort(__float2half(0));
94
-#endif
95
-
96
- lut_index1 = tmp1 & 0xF;
97
- lut_index2 = (tmp1 >> 4) & 0xF;
98
-#ifndef USE_ROCM
99
- tmp2.x = deq2[lut_index1][off];
100
- tmp2.y = deq2[lut_index2][off];
101
-#else
102
- tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
103
- tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
104
-#endif
105
- res2 = __hfma2(tmp2, blockvec[k + 0], res2);
106
-
107
- lut_index1 = (tmp1 >> 8) & 0xF;
108
- lut_index2 = (tmp1 >> 12) & 0xF;
109
-#ifndef USE_ROCM
110
- tmp2.x = deq2[lut_index1][off];
111
- tmp2.y = deq2[lut_index2][off];
112
-#else
113
- tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
114
- tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
115
-#endif
116
- res2 = __hfma2(tmp2, blockvec[k + 1], res2);
117
-
118
- lut_index1 = (tmp1 >> 16) & 0xF;
119
- lut_index2 = (tmp1 >> 20) & 0xF;
120
-#ifndef USE_ROCM
121
- tmp2.x = deq2[lut_index1][off];
122
- tmp2.y = deq2[lut_index2][off];
123
-#else
124
- tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
125
- tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
126
-#endif
127
- res2 = __hfma2(tmp2, blockvec[k + 2], res2);
128
-
129
- lut_index1 = (tmp1 >> 24) & 0xF;
130
- lut_index2 = (tmp1 >> 28) & 0xF;
131
-#ifndef USE_ROCM
132
- tmp2.x = deq2[lut_index1][off];
133
- tmp2.y = deq2[lut_index2][off];
134
-#else
135
- tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
136
- tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
137
-#endif
138
- res2 = __hfma2(tmp2, blockvec[k + 3], res2);
139
-
140
-#ifndef USE_ROCM
141
- res = __hadd(__hadd(res2.x, res2.y), res);
142
-#else
143
- res = __hadd(__hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)),
144
- res);
145
-#endif
146
-
147
- i += width;
148
- k += 4;
149
- }
150
-
151
- // col%2 -> only set one of the two values
152
-#ifndef USE_ROCM
153
- half2 res3 = {};
154
- if (col % 2 == 0) {
155
- res3.x = res;
156
- } else {
157
- res3.y = res;
158
- }
159
-#else
160
- __half2 res3;
161
- res3.x = __half_as_ushort(__float2half(0));
162
- res3.y = __half_as_ushort(__float2half(0));
163
- if (col % 2 == 0) {
164
- res3.x = __half_as_ushort(res);
165
- } else {
166
- res3.y = __half_as_ushort(res);
167
- }
168
-#endif
169
-
170
-#ifndef USE_ROCM
171
- atomicAdd(&mul[b * width / 2 + col / 2], res3);
172
-#else
173
- int tmp_addr = b * width / 2 + col / 2;
174
- atomicAdd(&(mul[tmp_addr].x), __half2float(__ushort_as_half(res3.x)));
175
- atomicAdd(&(mul[tmp_addr].y), __half2float(__ushort_as_half(res3.y)));
176
-#endif
177
- }
178
-}
179
-
180
-} // namespace squeezellm
181
-} // namespace vllm
182
-
183
-// 4-bit matvec kernel (LUT-based)
184
-void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
185
- torch::Tensor lookup_table) {
186
- int height = mat.size(0);
187
- int width = mat.size(1);
188
-
189
- int batch = vec.size(0);
190
- int vec_height = vec.size(1);
191
-
192
- dim3 blocks((height + BLOCKHEIGHT4 - 1) / BLOCKHEIGHT4,
193
- (width + BLOCKWIDTH - 1) / BLOCKWIDTH);
194
- dim3 threads(BLOCKWIDTH);
195
-
196
- const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));
197
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
198
- vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads, 0, stream>>>(
199
-#ifndef USE_ROCM
200
- (half2*)vec.data_ptr<at::Half>(),
201
-#else
202
- (__half2*)vec.data_ptr<at::Half>(),
203
-#endif
204
- mat.data_ptr<int>(),
205
-#ifndef USE_ROCM
206
- (half2*)mul.data_ptr<at::Half>(),
207
- (__half*)lookup_table.data_ptr<at::Half>(),
208
-#else
209
- (float2*)mul.data_ptr<float>(),
210
- (__half*)lookup_table.data_ptr<at::Half>(),
211
-#endif
212
- height, width, batch, vec_height);
213
-}
214
-
215
-#undef BLOCKWIDTH
216
-#undef BLOCKHEIGHT4
csrc/torch_bindings.cpp
-6
@@ -237,12 +237,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
237
ops.def("gptq_shuffle(Tensor! q_weight, Tensor q_perm, int bit) -> ()");
238
ops.impl("gptq_shuffle", torch::kCUDA, &gptq_shuffle);
239
240
- // Quantized GEMM for SqueezeLLM.
241
- ops.def(
242
- "squeezellm_gemm(Tensor vec, Tensor mat, Tensor! mul, Tensor "
243
- "lookup_table) -> ()");
244
- ops.impl("squeezellm_gemm", torch::kCUDA, &squeezellm_gemm);
245
-
240
// Compute FP8 quantized tensor for given scaling factor.
241
ops.def(
242
"static_scaled_fp8_quant(Tensor! out, Tensor input, Tensor scale) -> ()");
docs/source/quantization/supported_hardware.rst
-11
@@ -119,17 +119,6 @@ The table below shows the compatibility of various quantization implementations
119
- ✗
120
- ✗
121
- ✗
122
- * - SqueezeLLM
123
- - ✅︎
124
- - ✅︎
125
- - ✅︎
126
- - ✅︎
127
- - ✅︎
128
- - ✗
129
- - ✗
130
- - ✗
131
- - ✗
132
- - ✗
122
123
Notes:
124
^^^^^^
examples/fp8/README.md
+2
-2
@@ -62,7 +62,7 @@ This script evaluates the inference throughput of language models using various
62
63
python3 benchmarks/benchmark_throughput.py --help
64
usage: benchmark_throughput.py [-h] [--backend {vllm,hf,mii}] [--dataset DATASET] [--input-len INPUT_LEN] [--output-len OUTPUT_LEN] [--model MODEL]
65
- [--tokenizer TOKENIZER] [--quantization {awq,gptq,squeezellm,None}] [--tensor-parallel-size TENSOR_PARALLEL_SIZE] [--n N]
65
+ [--tokenizer TOKENIZER] [--quantization {awq,gptq,None}] [--tensor-parallel-size TENSOR_PARALLEL_SIZE] [--n N]
66
[--use-beam-search] [--num-prompts NUM_PROMPTS] [--seed SEED] [--hf-max-batch-size HF_MAX_BATCH_SIZE] [--trust-remote-code]
67
[--max-model-len MAX_MODEL_LEN] [--dtype {auto,half,float16,bfloat16,float,float32}] [--enforce-eager] [--kv-cache-dtype {auto,fp8}]
68
[--quantization-param-path KV_CACHE_quantization_param_path]
@@ -76,7 +76,7 @@ optional arguments:
76
--output-len OUTPUT_LEN Output length for each request. Overrides the output length from the dataset.
77
--model MODEL
78
--tokenizer TOKENIZER
79
- --quantization {awq,gptq,squeezellm,None}, -q {awq,gptq,squeezellm,None}
79
+ --quantization {awq,gptq,None}, -q {awq,gptq,None}
80
--tensor-parallel-size TENSOR_PARALLEL_SIZE, -tp TENSOR_PARALLEL_SIZE
81
--n N Number of generated sequences per prompt.
82
--use-beam-search
vllm/_custom_ops.py
-6
@@ -209,12 +209,6 @@ def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor,
209
torch.ops._C.gptq_shuffle(q_weight, q_perm, bit)
210
211
212
-# squeezellm
213
-def squeezellm_gemm(vec: torch.Tensor, mat: torch.Tensor, mul: torch.Tensor,
214
- lookup_table: torch.Tensor) -> None:
215
- torch.ops._C.squeezellm_gemm(vec, mat, mul, lookup_table)
216
-
217
-
212
# marlin
213
def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
214
b_scales: torch.Tensor, workspace: torch.Tensor, size_m: int,
vllm/config.py
+2
-2
@@ -277,7 +277,7 @@ class ModelConfig:
277
278
def _verify_quantization(self) -> None:
279
supported_quantization = [*QUANTIZATION_METHODS]
280
- rocm_supported_quantization = ["awq", "gptq", "squeezellm", "fp8"]
280
+ rocm_supported_quantization = ["awq", "gptq", "fp8"]
281
optimized_quantization_methods = [
282
"fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
283
"fbgemm_fp8", "compressed_tensors", "compressed-tensors",
@@ -1537,7 +1537,7 @@ class LoRAConfig:
1537
if model_config.quantization and model_config.quantization not in [
1538
"awq", "gptq"
1539
]:
1540
- # TODO support marlin and squeezellm
1540
+ # TODO support marlin
1541
logger.warning("%s quantization is not tested with LoRA yet.",
1542
model_config.quantization)
1543
vllm/entrypoints/llm.py
+1
-1
@@ -55,7 +55,7 @@ class LLM:
55
However, if the `torch_dtype` in the config is `float32`, we will
56
use `float16` instead.
57
quantization: The method used to quantize the model weights. Currently,
58
- we support "awq", "gptq", "squeezellm", and "fp8" (experimental).
58
+ we support "awq", "gptq", and "fp8" (experimental).
59
If None, we first check the `quantization_config` attribute in the
60
model config file. If that is None, we assume the model weights are
61
not quantized and use `dtype` to determine the data type of
vllm/lora/layers.py
+1
-1
@@ -39,7 +39,7 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
39
# unquantizedLinear
40
if hasattr(base_layer, "weight"):
41
return base_layer.weight.device
42
- # GPTQ/AWQ/SqueezeLLM
42
+ # GPTQ/AWQ
43
elif hasattr(base_layer, "qweight"):
44
return base_layer.qweight.device
45
# marlin
vllm/model_executor/layers/quantization/__init__.py
-2
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.quantization.marlin import MarlinConfig
25
from vllm.model_executor.layers.quantization.neuron_quant import (
26
NeuronQuantConfig)
27
from vllm.model_executor.layers.quantization.qqq import QQQConfig
28
-from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
28
from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig
29
30
QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
@@ -43,7 +42,6 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
42
"gptq_marlin": GPTQMarlinConfig,
43
"awq_marlin": AWQMarlinConfig,
44
"gptq": GPTQConfig,
46
- "squeezellm": SqueezeLLMConfig,
45
"compressed-tensors": CompressedTensorsConfig,
46
"bitsandbytes": BitsAndBytesConfig,
47
"qqq": QQQConfig,
vllm/model_executor/layers/quantization/squeezellm.py
deleted
-138
@@ -1,138 +0,0 @@
1
-from typing import Any, Dict, List, Optional
2
-
3
-import torch
4
-from torch.nn.parameter import Parameter
5
-
6
-from vllm import _custom_ops as ops
7
-from vllm.model_executor.layers.linear import LinearBase
8
-from vllm.model_executor.layers.quantization.base_config import (
9
- QuantizationConfig, QuantizeMethodBase)
10
-from vllm.model_executor.utils import set_weight_attrs
11
-from vllm.utils import is_hip
12
-
13
-
14
-class SqueezeLLMConfig(QuantizationConfig):
15
- """Config class for SqueezeLLM.
16
-
17
- Reference: https://arxiv.org/pdf/2306.07629
18
- """
19
-
20
- def __init__(
21
- self,
22
- weight_bits: int,
23
- ) -> None:
24
- self.weight_bits = weight_bits
25
-
26
- if self.weight_bits != 4:
27
- raise ValueError(
28
- "Currently, only 4-bit weight quantization is supported for "
29
- f"SqueezeLLM, but got {self.weight_bits} bits.")
30
-
31
- self.pack_factor = 32 // self.weight_bits
32
-
33
- def __repr__(self) -> str:
34
- return f"SqueezeLLMConfig(weight_bits={self.weight_bits})"
35
-
36
- def get_name(self) -> str:
37
- return "squeezellm"
38
-
39
- def get_supported_act_dtypes(self) -> List[torch.dtype]:
40
- return [torch.half]
41
-
42
- @classmethod
43
- def get_min_capability(cls) -> int:
44
- return 70
45
-
46
- @staticmethod
47
- def get_config_filenames() -> List[str]:
48
- return ["quant_config.json"]
49
-
50
- @classmethod
51
- def from_config(cls, config: Dict[str, Any]) -> "SqueezeLLMConfig":
52
- weight_bits = cls.get_from_keys(config, ["wbits"])
53
- return cls(weight_bits)
54
-
55
- def get_quant_method(self, layer: torch.nn.Module,
56
- prefix: str) -> Optional[QuantizeMethodBase]:
57
- if isinstance(layer, LinearBase):
58
- return SqueezeLLMLinearMethod(self)
59
- return None
60
-
61
- def get_scaled_act_names(self) -> List[str]:
62
- return []
63
-
64
-
65
-class SqueezeLLMLinearMethod(QuantizeMethodBase):
66
- """Linear method for SqueezeLLM.
67
-
68
- Args:
69
- quant_config: The SqueezeLLM quantization config.
70
- """
71
-
72
- def __init__(self, quant_config: SqueezeLLMConfig):
73
- self.quant_config = quant_config
74
-
75
- def create_weights(self, layer: torch.nn.Module,
76
- input_size_per_partition: int,
77
- output_partition_sizes: List[int], input_size: int,
78
- output_size: int, params_dtype: torch.dtype,
79
- **extra_weight_attrs):
80
- if input_size_per_partition % self.quant_config.pack_factor != 0:
81
- raise ValueError(
82
- "The input size is not aligned with the quantized "
83
- "weight shape. This can be caused by too large "
84
- "tensor parallel size.")
85
-
86
- output_size_per_partition = sum(output_partition_sizes)
87
- qweight = Parameter(
88
- torch.empty(
89
- input_size_per_partition // self.quant_config.pack_factor,
90
- output_size_per_partition,
91
- dtype=torch.int32,
92
- ),
93
- requires_grad=False,
94
- )
95
- set_weight_attrs(
96
- qweight, {
97
- "input_dim": 0,
98
- "output_dim": 1,
99
- "packed_dim": 0,
100
- "pack_factor": self.quant_config.pack_factor,
101
- })
102
- lookup_table = Parameter(
103
- torch.empty(
104
- output_size,
105
- self.quant_config.weight_bits**2,
106
- dtype=params_dtype,
107
- ),
108
- requires_grad=False,
109
- )
110
- set_weight_attrs(lookup_table, {
111
- "output_dim": 0,
112
- })
113
-
114
- layer.register_parameter("qweight", qweight)
115
- set_weight_attrs(qweight, extra_weight_attrs)
116
- layer.register_parameter("lookup_table", lookup_table)
117
- set_weight_attrs(lookup_table, extra_weight_attrs)
118
-
119
- def apply(self,
120
- layer: torch.nn.Module,
121
- x: torch.Tensor,
122
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
123
- qweight = layer.qweight
124
- lookup_table = layer.lookup_table
125
- out_shape = x.shape[:-1] + (qweight.shape[-1], )
126
- reshaped_x = x.reshape(-1, x.shape[-1])
127
- if is_hip():
128
- out_f = torch.zeros(out_shape, dtype=torch.float)
129
- ops.squeezellm_gemm(reshaped_x, qweight, out_f, lookup_table)
130
- out = out_f.to(dtype=torch.float16)
131
- else:
132
- # NOTE: The output tensor should be zero-initialized.
133
- out = torch.zeros(out_shape, dtype=torch.float16)
134
- ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
135
-
136
- if bias is not None:
137
- out.add_(bias)
138
- return out.reshape(out_shape)