@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 23f322297f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Remove `SqueezeLLM` (#8220)

Dipika Sikka committed Sep 6, 2024 at 18:29 UTC 23f322297f33a50dd1fe0870665d0c4414fd78ab
12 files changed +6 -389
CMakeLists.txt
-1
@@ -181,7 +181,6 @@ set(VLLM_EXT_SRC
181 "csrc/pos_encoding_kernels.cu"
182 "csrc/activation_kernels.cu"
183 "csrc/layernorm_kernels.cu"
184 - "csrc/quantization/squeezellm/quant_cuda_kernel.cu"
184 "csrc/quantization/gptq/q_gemm.cu"
185 "csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
186 "csrc/quantization/fp8/common.cu"
csrc/ops.h
-3
@@ -170,9 +170,6 @@ void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
170 void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
171 torch::Tensor& scales);
172
173 -void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
174 - torch::Tensor lookup_table);
175 -
173 torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
174 torch::Tensor b_gptq_qzeros,
175 torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
csrc/quantization/squeezellm/quant_cuda_kernel.cu deleted
-216
@@ -1,216 +0,0 @@
1 -#include <torch/all.h>
2 -#include <cuda.h>
3 -#include <cuda_runtime.h>
4 -#include <cuda_fp16.h>
5 -
6 -// half-tensor
7 -#include <c10/cuda/CUDAStream.h>
8 -#include <ATen/cuda/CUDATensorMethods.cuh>
9 -#include <c10/cuda/CUDAGuard.h>
10 -
11 -#define BLOCKWIDTH 128
12 -#define BLOCKHEIGHT4 16
13 -
14 -namespace vllm {
15 -namespace squeezellm {
16 -
17 -__device__ inline unsigned int as_unsigned(int i) {
18 - return *reinterpret_cast<unsigned int*>(&i);
19 -}
20 -
21 -// 4-bit matvec kernel (LUT-based)
22 -__global__ void NUQ4MatMulKernel(
23 -#ifndef USE_ROCM
24 - const half2* __restrict__ vec,
25 -#else
26 - const __half2* __restrict__ vec,
27 -#endif
28 - const int* __restrict__ mat,
29 -#ifndef USE_ROCM
30 - half2* __restrict__ mul,
31 -#else
32 - float2* __restrict__ mul,
33 -#endif
34 - const __half* __restrict__ lookup_table, int height, int width, int batch,
35 - int vec_height) {
36 -
37 - const int blockwidth2 = BLOCKWIDTH / 2;
38 -
39 - int row = BLOCKHEIGHT4 * blockIdx.x;
40 - int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
41 -
42 -#ifndef USE_ROCM
43 - __shared__ half2 blockvec[blockwidth2];
44 -#else
45 - __shared__ __half2 blockvec[blockwidth2];
46 -#endif
47 -
48 - __shared__ __half deq2[16][BLOCKWIDTH];
49 - int off = threadIdx.x;
50 - int column_offset = col * 16;
51 - for (int val = 0; val < 16; val += 1) {
52 - int lut_index = column_offset + val;
53 - deq2[val][off] = lookup_table[lut_index];
54 - }
55 -
56 - __half res;
57 -#ifndef USE_ROCM
58 - half2 res2;
59 - half2 tmp2;
60 -#else
61 - __half2 res2;
62 - __half2 tmp2;
63 -#endif
64 -
65 - int i;
66 - int k;
67 -
68 - unsigned int tmp1;
69 - unsigned int lut_index1, lut_index2;
70 -
71 - for (int b = 0; b < batch; ++b) {
72 - i = width * row + col;
73 - res = __int2half_rd(0);
74 - k = 0;
75 -
76 - __syncthreads();
77 - if (threadIdx.x < blockwidth2)
78 - blockvec[threadIdx.x] =
79 - vec[b * vec_height / 2 + (row / BLOCKHEIGHT4) * blockwidth2 +
80 - threadIdx.x];
81 - __syncthreads();
82 -
83 - while (k < blockwidth2) {
84 - tmp1 = as_unsigned(mat[i]);
85 -
86 -#ifndef USE_ROCM
87 - res2 = {};
88 - tmp2 = {};
89 -#else
90 - res2.x = __half_as_ushort(__float2half(0));
91 - res2.y = __half_as_ushort(__float2half(0));
92 - tmp2.x = __half_as_ushort(__float2half(0));
93 - tmp2.y = __half_as_ushort(__float2half(0));
94 -#endif
95 -
96 - lut_index1 = tmp1 & 0xF;
97 - lut_index2 = (tmp1 >> 4) & 0xF;
98 -#ifndef USE_ROCM
99 - tmp2.x = deq2[lut_index1][off];
100 - tmp2.y = deq2[lut_index2][off];
101 -#else
102 - tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
103 - tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
104 -#endif
105 - res2 = __hfma2(tmp2, blockvec[k + 0], res2);
106 -
107 - lut_index1 = (tmp1 >> 8) & 0xF;
108 - lut_index2 = (tmp1 >> 12) & 0xF;
109 -#ifndef USE_ROCM
110 - tmp2.x = deq2[lut_index1][off];
111 - tmp2.y = deq2[lut_index2][off];
112 -#else
113 - tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
114 - tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
115 -#endif
116 - res2 = __hfma2(tmp2, blockvec[k + 1], res2);
117 -
118 - lut_index1 = (tmp1 >> 16) & 0xF;
119 - lut_index2 = (tmp1 >> 20) & 0xF;
120 -#ifndef USE_ROCM
121 - tmp2.x = deq2[lut_index1][off];
122 - tmp2.y = deq2[lut_index2][off];
123 -#else
124 - tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
125 - tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
126 -#endif
127 - res2 = __hfma2(tmp2, blockvec[k + 2], res2);
128 -
129 - lut_index1 = (tmp1 >> 24) & 0xF;
130 - lut_index2 = (tmp1 >> 28) & 0xF;
131 -#ifndef USE_ROCM
132 - tmp2.x = deq2[lut_index1][off];
133 - tmp2.y = deq2[lut_index2][off];
134 -#else
135 - tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
136 - tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
137 -#endif
138 - res2 = __hfma2(tmp2, blockvec[k + 3], res2);
139 -
140 -#ifndef USE_ROCM
141 - res = __hadd(__hadd(res2.x, res2.y), res);
142 -#else
143 - res = __hadd(__hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)),
144 - res);
145 -#endif
146 -
147 - i += width;
148 - k += 4;
149 - }
150 -
151 - // col%2 -> only set one of the two values
152 -#ifndef USE_ROCM
153 - half2 res3 = {};
154 - if (col % 2 == 0) {
155 - res3.x = res;
156 - } else {
157 - res3.y = res;
158 - }
159 -#else
160 - __half2 res3;
161 - res3.x = __half_as_ushort(__float2half(0));
162 - res3.y = __half_as_ushort(__float2half(0));
163 - if (col % 2 == 0) {
164 - res3.x = __half_as_ushort(res);
165 - } else {
166 - res3.y = __half_as_ushort(res);
167 - }
168 -#endif
169 -
170 -#ifndef USE_ROCM
171 - atomicAdd(&mul[b * width / 2 + col / 2], res3);
172 -#else
173 - int tmp_addr = b * width / 2 + col / 2;
174 - atomicAdd(&(mul[tmp_addr].x), __half2float(__ushort_as_half(res3.x)));
175 - atomicAdd(&(mul[tmp_addr].y), __half2float(__ushort_as_half(res3.y)));
176 -#endif
177 - }
178 -}
179 -
180 -} // namespace squeezellm
181 -} // namespace vllm
182 -
183 -// 4-bit matvec kernel (LUT-based)
184 -void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
185 - torch::Tensor lookup_table) {
186 - int height = mat.size(0);
187 - int width = mat.size(1);
188 -
189 - int batch = vec.size(0);
190 - int vec_height = vec.size(1);
191 -
192 - dim3 blocks((height + BLOCKHEIGHT4 - 1) / BLOCKHEIGHT4,
193 - (width + BLOCKWIDTH - 1) / BLOCKWIDTH);
194 - dim3 threads(BLOCKWIDTH);
195 -
196 - const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));
197 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
198 - vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads, 0, stream>>>(
199 -#ifndef USE_ROCM
200 - (half2*)vec.data_ptr<at::Half>(),
201 -#else
202 - (__half2*)vec.data_ptr<at::Half>(),
203 -#endif
204 - mat.data_ptr<int>(),
205 -#ifndef USE_ROCM
206 - (half2*)mul.data_ptr<at::Half>(),
207 - (__half*)lookup_table.data_ptr<at::Half>(),
208 -#else
209 - (float2*)mul.data_ptr<float>(),
210 - (__half*)lookup_table.data_ptr<at::Half>(),
211 -#endif
212 - height, width, batch, vec_height);
213 -}
214 -
215 -#undef BLOCKWIDTH
216 -#undef BLOCKHEIGHT4
csrc/torch_bindings.cpp
-6
@@ -237,12 +237,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
237 ops.def("gptq_shuffle(Tensor! q_weight, Tensor q_perm, int bit) -> ()");
238 ops.impl("gptq_shuffle", torch::kCUDA, &gptq_shuffle);
239
240 - // Quantized GEMM for SqueezeLLM.
241 - ops.def(
242 - "squeezellm_gemm(Tensor vec, Tensor mat, Tensor! mul, Tensor "
243 - "lookup_table) -> ()");
244 - ops.impl("squeezellm_gemm", torch::kCUDA, &squeezellm_gemm);
245 -
240 // Compute FP8 quantized tensor for given scaling factor.
241 ops.def(
242 "static_scaled_fp8_quant(Tensor! out, Tensor input, Tensor scale) -> ()");
docs/source/quantization/supported_hardware.rst
-11
@@ -119,17 +119,6 @@ The table below shows the compatibility of various quantization implementations
119 - ✗
120 - ✗
121 - ✗
122 - * - SqueezeLLM
123 - - ✅︎
124 - - ✅︎
125 - - ✅︎
126 - - ✅︎
127 - - ✅︎
128 - - ✗
129 - - ✗
130 - - ✗
131 - - ✗
132 - - ✗
122
123 Notes:
124 ^^^^^^
examples/fp8/README.md
+2 -2
@@ -62,7 +62,7 @@ This script evaluates the inference throughput of language models using various
62
63 python3 benchmarks/benchmark_throughput.py --help
64 usage: benchmark_throughput.py [-h] [--backend {vllm,hf,mii}] [--dataset DATASET] [--input-len INPUT_LEN] [--output-len OUTPUT_LEN] [--model MODEL]
65 - [--tokenizer TOKENIZER] [--quantization {awq,gptq,squeezellm,None}] [--tensor-parallel-size TENSOR_PARALLEL_SIZE] [--n N]
65 + [--tokenizer TOKENIZER] [--quantization {awq,gptq,None}] [--tensor-parallel-size TENSOR_PARALLEL_SIZE] [--n N]
66 [--use-beam-search] [--num-prompts NUM_PROMPTS] [--seed SEED] [--hf-max-batch-size HF_MAX_BATCH_SIZE] [--trust-remote-code]
67 [--max-model-len MAX_MODEL_LEN] [--dtype {auto,half,float16,bfloat16,float,float32}] [--enforce-eager] [--kv-cache-dtype {auto,fp8}]
68 [--quantization-param-path KV_CACHE_quantization_param_path]
@@ -76,7 +76,7 @@ optional arguments:
76 --output-len OUTPUT_LEN Output length for each request. Overrides the output length from the dataset.
77 --model MODEL
78 --tokenizer TOKENIZER
79 - --quantization {awq,gptq,squeezellm,None}, -q {awq,gptq,squeezellm,None}
79 + --quantization {awq,gptq,None}, -q {awq,gptq,None}
80 --tensor-parallel-size TENSOR_PARALLEL_SIZE, -tp TENSOR_PARALLEL_SIZE
81 --n N Number of generated sequences per prompt.
82 --use-beam-search
vllm/_custom_ops.py
-6
@@ -209,12 +209,6 @@ def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor,
209 torch.ops._C.gptq_shuffle(q_weight, q_perm, bit)
210
211
212 -# squeezellm
213 -def squeezellm_gemm(vec: torch.Tensor, mat: torch.Tensor, mul: torch.Tensor,
214 - lookup_table: torch.Tensor) -> None:
215 - torch.ops._C.squeezellm_gemm(vec, mat, mul, lookup_table)
216 -
217 -
212 # marlin
213 def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
214 b_scales: torch.Tensor, workspace: torch.Tensor, size_m: int,
vllm/config.py
+2 -2
@@ -277,7 +277,7 @@ class ModelConfig:
277
278 def _verify_quantization(self) -> None:
279 supported_quantization = [*QUANTIZATION_METHODS]
280 - rocm_supported_quantization = ["awq", "gptq", "squeezellm", "fp8"]
280 + rocm_supported_quantization = ["awq", "gptq", "fp8"]
281 optimized_quantization_methods = [
282 "fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
283 "fbgemm_fp8", "compressed_tensors", "compressed-tensors",
@@ -1537,7 +1537,7 @@ class LoRAConfig:
1537 if model_config.quantization and model_config.quantization not in [
1538 "awq", "gptq"
1539 ]:
1540 - # TODO support marlin and squeezellm
1540 + # TODO support marlin
1541 logger.warning("%s quantization is not tested with LoRA yet.",
1542 model_config.quantization)
1543
vllm/entrypoints/llm.py
+1 -1
@@ -55,7 +55,7 @@ class LLM:
55 However, if the `torch_dtype` in the config is `float32`, we will
56 use `float16` instead.
57 quantization: The method used to quantize the model weights. Currently,
58 - we support "awq", "gptq", "squeezellm", and "fp8" (experimental).
58 + we support "awq", "gptq", and "fp8" (experimental).
59 If None, we first check the `quantization_config` attribute in the
60 model config file. If that is None, we assume the model weights are
61 not quantized and use `dtype` to determine the data type of
vllm/lora/layers.py
+1 -1
@@ -39,7 +39,7 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
39 # unquantizedLinear
40 if hasattr(base_layer, "weight"):
41 return base_layer.weight.device
42 - # GPTQ/AWQ/SqueezeLLM
42 + # GPTQ/AWQ
43 elif hasattr(base_layer, "qweight"):
44 return base_layer.qweight.device
45 # marlin
vllm/model_executor/layers/quantization/__init__.py
-2
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.quantization.marlin import MarlinConfig
25 from vllm.model_executor.layers.quantization.neuron_quant import (
26 NeuronQuantConfig)
27 from vllm.model_executor.layers.quantization.qqq import QQQConfig
28 -from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
28 from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig
29
30 QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
@@ -43,7 +42,6 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
42 "gptq_marlin": GPTQMarlinConfig,
43 "awq_marlin": AWQMarlinConfig,
44 "gptq": GPTQConfig,
46 - "squeezellm": SqueezeLLMConfig,
45 "compressed-tensors": CompressedTensorsConfig,
46 "bitsandbytes": BitsAndBytesConfig,
47 "qqq": QQQConfig,
vllm/model_executor/layers/quantization/squeezellm.py deleted
-138
@@ -1,138 +0,0 @@
1 -from typing import Any, Dict, List, Optional
2 -
3 -import torch
4 -from torch.nn.parameter import Parameter
5 -
6 -from vllm import _custom_ops as ops
7 -from vllm.model_executor.layers.linear import LinearBase
8 -from vllm.model_executor.layers.quantization.base_config import (
9 - QuantizationConfig, QuantizeMethodBase)
10 -from vllm.model_executor.utils import set_weight_attrs
11 -from vllm.utils import is_hip
12 -
13 -
14 -class SqueezeLLMConfig(QuantizationConfig):
15 - """Config class for SqueezeLLM.
16 -
17 - Reference: https://arxiv.org/pdf/2306.07629
18 - """
19 -
20 - def __init__(
21 - self,
22 - weight_bits: int,
23 - ) -> None:
24 - self.weight_bits = weight_bits
25 -
26 - if self.weight_bits != 4:
27 - raise ValueError(
28 - "Currently, only 4-bit weight quantization is supported for "
29 - f"SqueezeLLM, but got {self.weight_bits} bits.")
30 -
31 - self.pack_factor = 32 // self.weight_bits
32 -
33 - def __repr__(self) -> str:
34 - return f"SqueezeLLMConfig(weight_bits={self.weight_bits})"
35 -
36 - def get_name(self) -> str:
37 - return "squeezellm"
38 -
39 - def get_supported_act_dtypes(self) -> List[torch.dtype]:
40 - return [torch.half]
41 -
42 - @classmethod
43 - def get_min_capability(cls) -> int:
44 - return 70
45 -
46 - @staticmethod
47 - def get_config_filenames() -> List[str]:
48 - return ["quant_config.json"]
49 -
50 - @classmethod
51 - def from_config(cls, config: Dict[str, Any]) -> "SqueezeLLMConfig":
52 - weight_bits = cls.get_from_keys(config, ["wbits"])
53 - return cls(weight_bits)
54 -
55 - def get_quant_method(self, layer: torch.nn.Module,
56 - prefix: str) -> Optional[QuantizeMethodBase]:
57 - if isinstance(layer, LinearBase):
58 - return SqueezeLLMLinearMethod(self)
59 - return None
60 -
61 - def get_scaled_act_names(self) -> List[str]:
62 - return []
63 -
64 -
65 -class SqueezeLLMLinearMethod(QuantizeMethodBase):
66 - """Linear method for SqueezeLLM.
67 -
68 - Args:
69 - quant_config: The SqueezeLLM quantization config.
70 - """
71 -
72 - def __init__(self, quant_config: SqueezeLLMConfig):
73 - self.quant_config = quant_config
74 -
75 - def create_weights(self, layer: torch.nn.Module,
76 - input_size_per_partition: int,
77 - output_partition_sizes: List[int], input_size: int,
78 - output_size: int, params_dtype: torch.dtype,
79 - **extra_weight_attrs):
80 - if input_size_per_partition % self.quant_config.pack_factor != 0:
81 - raise ValueError(
82 - "The input size is not aligned with the quantized "
83 - "weight shape. This can be caused by too large "
84 - "tensor parallel size.")
85 -
86 - output_size_per_partition = sum(output_partition_sizes)
87 - qweight = Parameter(
88 - torch.empty(
89 - input_size_per_partition // self.quant_config.pack_factor,
90 - output_size_per_partition,
91 - dtype=torch.int32,
92 - ),
93 - requires_grad=False,
94 - )
95 - set_weight_attrs(
96 - qweight, {
97 - "input_dim": 0,
98 - "output_dim": 1,
99 - "packed_dim": 0,
100 - "pack_factor": self.quant_config.pack_factor,
101 - })
102 - lookup_table = Parameter(
103 - torch.empty(
104 - output_size,
105 - self.quant_config.weight_bits**2,
106 - dtype=params_dtype,
107 - ),
108 - requires_grad=False,
109 - )
110 - set_weight_attrs(lookup_table, {
111 - "output_dim": 0,
112 - })
113 -
114 - layer.register_parameter("qweight", qweight)
115 - set_weight_attrs(qweight, extra_weight_attrs)
116 - layer.register_parameter("lookup_table", lookup_table)
117 - set_weight_attrs(lookup_table, extra_weight_attrs)
118 -
119 - def apply(self,
120 - layer: torch.nn.Module,
121 - x: torch.Tensor,
122 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
123 - qweight = layer.qweight
124 - lookup_table = layer.lookup_table
125 - out_shape = x.shape[:-1] + (qweight.shape[-1], )
126 - reshaped_x = x.reshape(-1, x.shape[-1])
127 - if is_hip():
128 - out_f = torch.zeros(out_shape, dtype=torch.float)
129 - ops.squeezellm_gemm(reshaped_x, qweight, out_f, lookup_table)
130 - out = out_f.to(dtype=torch.float16)
131 - else:
132 - # NOTE: The output tensor should be zero-initialized.
133 - out = torch.zeros(out_shape, dtype=torch.float16)
134 - ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
135 -
136 - if bias is not None:
137 - out.add_(bias)
138 - return out.reshape(out_shape)