A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Initial Activation Quantization Support (#4525)
Co-authored-by: Varun Sundar Rabindranath <varunsundar08@gmail.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Dipika Sikka committed
May 23, 2024 at 17:29 UTC
a1242324c99ff8b1e29981006dfb504da198c7c3
17 files changed
+683
-94
CMakeLists.txt
+1
@@ -167,6 +167,7 @@ set(VLLM_EXT_SRC
167
"csrc/layernorm_kernels.cu"
168
"csrc/quantization/squeezellm/quant_cuda_kernel.cu"
169
"csrc/quantization/gptq/q_gemm.cu"
170
+ "csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
171
"csrc/quantization/fp8/common.cu"
172
"csrc/cuda_utils_kernels.cu"
173
"csrc/moe_align_block_size_kernels.cu"
csrc/ops.h
+3
@@ -93,6 +93,9 @@ int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
93
94
#endif
95
96
+void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor& input,
97
+ float scale);
98
+
99
void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
100
torch::Tensor lookup_table);
101
csrc/pybind.cpp
+3
@@ -67,6 +67,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
67
"Aligning the number of tokens to be processed by each expert such "
68
"that it is divisible by the block size.");
69
70
+ ops.def("static_scaled_int8_quant", &static_scaled_int8_quant,
71
+ "Compute int8 quantized tensor for given scaling factor");
72
+
73
// Cache ops
74
pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
75
cache_ops.def("swap_blocks", &swap_blocks,
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
new
+59
@@ -0,0 +1,59 @@
1
+#include <ATen/cuda/CUDAContext.h>
2
+#include <torch/extension.h>
3
+#include <cmath>
4
+
5
+#include "../../dispatch_utils.h"
6
+
7
+static inline __device__ int8_t float_to_int8_rn(float x) {
8
+#ifdef USE_ROCM
9
+ static const float i8_min =
10
+ static_cast<float>(std::numeric_limits<int8_t>::min());
11
+ static const float i8_max =
12
+ static_cast<float>(std::numeric_limits<int8_t>::max());
13
+ // round
14
+ float dst = std::nearbyint(x);
15
+ // saturate
16
+ dst = std::clamp(dst, i8_min, i8_max);
17
+ return static_cast<int8_t>(dst);
18
+#else
19
+ // CUDA path
20
+ uint32_t dst;
21
+ asm volatile("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(dst) : "f"(x));
22
+ return reinterpret_cast<const int8_t&>(dst);
23
+#endif
24
+}
25
+
26
+namespace vllm {
27
+
28
+template <typename scalar_t, typename scale_type>
29
+__global__ void static_scaled_int8_quant_kernel(
30
+ const scalar_t* __restrict__ input, int8_t* __restrict__ out,
31
+ scale_type scale, const int hidden_size) {
32
+ const int tid = threadIdx.x;
33
+ const int token_idx = blockIdx.x;
34
+
35
+ for (int i = tid; i < hidden_size; i += blockDim.x) {
36
+ out[token_idx * hidden_size + i] =
37
+ float_to_int8_rn(((float)input[token_idx * hidden_size + i]) / scale);
38
+ }
39
+}
40
+} // namespace vllm
41
+
42
+void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
43
+ torch::Tensor& input, // [..., hidden_size]
44
+ float scale) {
45
+ TORCH_CHECK(input.is_contiguous());
46
+ TORCH_CHECK(out.is_contiguous());
47
+ int hidden_size = input.size(-1);
48
+ int num_tokens = input.numel() / hidden_size;
49
+ dim3 grid(num_tokens);
50
+ dim3 block(std::min(hidden_size, 1024));
51
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
52
+ VLLM_DISPATCH_FLOATING_TYPES(
53
+ input.scalar_type(), "static_scaled_int8_quant_kernel", [&] {
54
+ vllm::static_scaled_int8_quant_kernel<scalar_t, float>
55
+ <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
56
+ out.data_ptr<int8_t>(), scale,
57
+ hidden_size);
58
+ });
59
+}
tests/kernels/test_int8_quant.py
new
+31
@@ -0,0 +1,31 @@
1
+import pytest
2
+import torch
3
+
4
+from vllm._C import ops
5
+
6
+DTYPES = [torch.half, torch.bfloat16, torch.float]
7
+HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 8192] # Arbitrary values for testing
8
+NUM_TOKENS = [1, 7, 83, 4096] # Arbitrary values for testing
9
+SEEDS = [0]
10
+SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
11
+
12
+
13
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
14
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
15
+@pytest.mark.parametrize("dtype", DTYPES)
16
+@pytest.mark.parametrize("seed", SEEDS)
17
+@pytest.mark.parametrize("scale", SCALE)
18
+@torch.inference_mode()
19
+def test_quant(num_tokens: int, hidden_size: int, dtype: torch.dtype,
20
+ seed: int, scale: float) -> None:
21
+ torch.random.manual_seed(seed)
22
+ torch.cuda.manual_seed(seed)
23
+ x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
24
+
25
+ out1 = (x / scale).round().clamp(
26
+ torch.iinfo(torch.int8).min,
27
+ torch.iinfo(torch.int8).max).to(torch.int8)
28
+ out2 = torch.empty_like(x, dtype=torch.int8)
29
+ ops.static_scaled_int8_quant(out2, x, scale)
30
+ assert torch.allclose(out1, out2,
31
+ atol=1) # big atol to account for rounding errors
tests/quantization/test_compressed_tensors.py
new
+36
@@ -0,0 +1,36 @@
1
+"""Test model set-up and weight loading for sparseml-quantized models.
2
+
3
+Run `pytest tests/quantization/test_compressed_tensors.py`.
4
+"""
5
+
6
+import torch
7
+
8
+from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
9
+ CompressedTensorsLinearMethod, CompressedTensorsW8A8StaticTensor)
10
+
11
+
12
+def test_compressed_tensors_w8a8_static_setup(vllm_runner):
13
+ model_path = "nm-testing/tinyllama-one-shot-static-quant-test-compressed"
14
+ llm = vllm_runner(model_path, quantization="sparseml", enforce_eager=True)
15
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
16
+ layer = model.model.layers[0]
17
+
18
+ qkv_proj = layer.self_attn.qkv_proj
19
+ o_proj = layer.self_attn.o_proj
20
+ gate_up_proj = layer.mlp.gate_up_proj
21
+ down_proj = layer.mlp.down_proj
22
+
23
+ assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
24
+ assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
25
+ assert isinstance(gate_up_proj.quant_method, CompressedTensorsLinearMethod)
26
+ assert isinstance(down_proj.quant_method, CompressedTensorsLinearMethod)
27
+
28
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
29
+
30
+ assert qkv_proj.weight.dtype is torch.int8
31
+ assert o_proj.weight.dtype is torch.int8
32
+ assert gate_up_proj.weight.dtype is torch.int8
33
+
34
+ assert qkv_proj.weight_scale.shard_splitter is not None
35
+ assert qkv_proj.weight_scale.logical_widths is not None
36
+ assert qkv_proj.input_scale.dtype is torch.float32
vllm/_custom_ops.py
+18
@@ -251,6 +251,24 @@ def scaled_fp8_quant(
251
return output, scale
252
253
254
+# int8
255
+def static_scaled_int8_quant(input: torch.Tensor,
256
+ scale: float) -> torch.Tensor:
257
+ """
258
+ Quantize the input tensor to int8 and return the quantized tensor.
259
+
260
+ Args:
261
+ input: The input tensor to be quantized to int8.
262
+ scale: Scaling factor for the int8 quantization.
263
+
264
+ Returns:
265
+ torch.Tensor: Output tensor in int8.
266
+ """
267
+ q = torch.empty_like(input, dtype=torch.int8)
268
+ vllm_ops.static_scaled_int8_quant(q, input, scale)
269
+ return q
270
+
271
+
272
# moe
273
def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
274
block_size: int, sorted_token_ids: torch.Tensor,
vllm/model_executor/layers/linear.py
+161
-83
@@ -56,7 +56,6 @@ class LinearMethodBase(QuantizeMethodBase):
56
x: torch.Tensor,
57
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
58
"""Apply the weights in layer to the input tensor.
59
-
59
Expects create_weights to have been called before on the layer."""
60
raise NotImplementedError
61
@@ -77,8 +76,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
76
output_partition_sizes: List[int], input_size: int,
77
output_size: int, params_dtype: torch.dtype,
78
**extra_weight_attrs):
80
- output_size_per_partition = sum(output_partition_sizes)
81
- weight = Parameter(torch.empty(output_size_per_partition,
79
+ weight = Parameter(torch.empty(sum(output_partition_sizes),
80
input_size_per_partition,
81
dtype=params_dtype),
82
requires_grad=False)
@@ -149,15 +147,13 @@ class ReplicatedLinear(LinearBase):
147
quant_config: Quantization configure.
148
"""
149
152
- def __init__(
153
- self,
154
- input_size: int,
155
- output_size: int,
156
- bias: bool = True,
157
- skip_bias_add: bool = False,
158
- params_dtype: Optional[torch.dtype] = None,
159
- quant_config: Optional[QuantizationConfig] = None,
160
- ):
150
+ def __init__(self,
151
+ input_size: int,
152
+ output_size: int,
153
+ bias: bool = True,
154
+ skip_bias_add: bool = False,
155
+ params_dtype: Optional[torch.dtype] = None,
156
+ quant_config: Optional[QuantizationConfig] = None):
157
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
158
quant_config)
159
@@ -210,17 +206,15 @@ class ColumnParallelLinear(LinearBase):
206
the list would be size 3.
207
"""
208
213
- def __init__(
214
- self,
215
- input_size: int,
216
- output_size: int,
217
- bias: bool = True,
218
- gather_output: bool = False,
219
- skip_bias_add: bool = False,
220
- params_dtype: Optional[torch.dtype] = None,
221
- quant_config: Optional[QuantizationConfig] = None,
222
- output_sizes: Optional[List[int]] = None,
223
- ):
209
+ def __init__(self,
210
+ input_size: int,
211
+ output_size: int,
212
+ bias: bool = True,
213
+ gather_output: bool = False,
214
+ skip_bias_add: bool = False,
215
+ params_dtype: Optional[torch.dtype] = None,
216
+ quant_config: Optional[QuantizationConfig] = None,
217
+ output_sizes: Optional[List[int]] = None):
218
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
219
quant_config)
220
@@ -228,18 +222,26 @@ class ColumnParallelLinear(LinearBase):
222
223
# Divide the weight matrix along the last dimension.
224
tp_size = get_tensor_model_parallel_world_size()
231
- self.output_size_per_partition = divide(output_size, tp_size)
225
+ assert self.quant_method is not None
226
+ self.output_size_per_partition = divide(self.output_size, tp_size)
227
+ self.output_partition_sizes = [self.output_size_per_partition]
228
+ # If QKV or MergedColumn, use output size of each partition.
229
+ if hasattr(self, "output_sizes"):
230
+ self.output_partition_sizes = [
231
+ divide(output_size, tp_size)
232
+ for output_size in self.output_sizes
233
+ ]
234
+
235
if output_sizes is None:
236
output_sizes = [output_size]
234
- # All the linear layer supports quant method.
235
- assert self.quant_method is not None
236
- self.quant_method.create_weights(self,
237
- self.input_size,
238
- [x // tp_size for x in output_sizes],
239
- self.input_size,
240
- self.output_size,
241
- self.params_dtype,
242
- weight_loader=self.weight_loader)
237
+ self.quant_method.create_weights(
238
+ layer=self,
239
+ input_size_per_partition=self.input_size,
240
+ output_partition_sizes=self.output_partition_sizes,
241
+ input_size=self.input_size,
242
+ output_size=self.output_size,
243
+ params_dtype=self.params_dtype,
244
+ weight_loader=self.weight_loader)
245
if bias:
246
self.bias = Parameter(
247
torch.empty(self.output_size_per_partition,
@@ -317,22 +319,24 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
319
quant_config: Quantization configure.
320
"""
321
320
- def __init__(
321
- self,
322
- input_size: int,
323
- output_sizes: List[int],
324
- bias: bool = True,
325
- gather_output: bool = False,
326
- skip_bias_add: bool = False,
327
- params_dtype: Optional[torch.dtype] = None,
328
- quant_config: Optional[QuantizationConfig] = None,
329
- ):
322
+ def __init__(self,
323
+ input_size: int,
324
+ output_sizes: List[int],
325
+ bias: bool = True,
326
+ gather_output: bool = False,
327
+ skip_bias_add: bool = False,
328
+ params_dtype: Optional[torch.dtype] = None,
329
+ quant_config: Optional[QuantizationConfig] = None):
330
self.output_sizes = output_sizes
331
tp_size = get_tensor_model_parallel_world_size()
332
assert all(output_size % tp_size == 0 for output_size in output_sizes)
333
- super().__init__(input_size, sum(output_sizes), bias, gather_output,
334
- skip_bias_add, params_dtype, quant_config,
335
- self.output_sizes)
333
+ super().__init__(input_size=input_size,
334
+ output_size=sum(output_sizes),
335
+ bias=bias,
336
+ gather_output=gather_output,
337
+ skip_bias_add=skip_bias_add,
338
+ params_dtype=params_dtype,
339
+ quant_config=quant_config)
340
341
def weight_loader(self,
342
param: Parameter,
@@ -343,6 +347,26 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
347
output_dim = getattr(param, "output_dim", None)
348
# Special case for AQLM codebooks.
349
is_metadata = getattr(param, "is_metadata", False)
350
+
351
+ param_shard_splitter = getattr(param, "shard_splitter", None)
352
+
353
+ if output_dim is not None and param_shard_splitter is not None:
354
+ raise NotImplementedError(
355
+ "We do not currently support output_dim != None and "
356
+ "shard_splitter != None for a parameter. Please open an issue."
357
+ )
358
+ # If a parameter has defined a shard_splitter to be used for
359
+ # the weight, it should be applied before the weight is
360
+ # loaded/copied to the parameter. The shard_splitter applies
361
+ # logic by using the loaded_shard_id to ensure that the loaded
362
+ # param is loaded to the correct location
363
+ # within the parameter defined by the linear method.
364
+ if loaded_shard_id is None and param_shard_splitter is not None:
365
+ raise NotImplementedError(
366
+ "We do not currently support loaded_shard_id == None and "
367
+ "shard_splitter != None for a parameter. Please open an issue."
368
+ )
369
+
370
# Special case for Fp8 scales.
371
fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
372
None)
@@ -403,6 +427,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
427
shard_size = loaded_weight.shape[0]
428
shard_offset = loaded_shard_id * shard_size
429
param_data = param_data.narrow(0, shard_offset, shard_size)
430
+
431
+ # If a param_shard_splitter is defined by the LinearMethod, use it.
432
+ elif param_shard_splitter is not None:
433
+ logical_widths = getattr(param, "logical_widths", None)
434
+ param_data, loaded_weight = param_shard_splitter(
435
+ param_data, loaded_weight, loaded_shard_id, logical_widths)
436
+
437
# Special case for Fp8 scales.
438
elif fp8_scales_shard_indexer is not None:
439
param_data, loaded_weight = fp8_scales_shard_indexer(
@@ -415,6 +446,14 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
446
"Loading a weight without `output_dim` attribute in "
447
"MergedColumnParallelLinear, assume the weight is "
448
"the same for all partitions.")
449
+
450
+ if fp8_scales_shard_indexer is None:
451
+ if len(param_data.shape) == 0:
452
+ param_data = param_data.reshape(1)
453
+
454
+ if len(loaded_weight.shape) == 0:
455
+ loaded_weight = loaded_weight.reshape(1)
456
+
457
assert param_data.shape == loaded_weight.shape
458
param_data.copy_(loaded_weight)
459
@@ -443,17 +482,15 @@ class QKVParallelLinear(ColumnParallelLinear):
482
quant_config: Quantization configure.
483
"""
484
446
- def __init__(
447
- self,
448
- hidden_size: int,
449
- head_size: int,
450
- total_num_heads: int,
451
- total_num_kv_heads: Optional[int] = None,
452
- bias: bool = True,
453
- skip_bias_add: bool = False,
454
- params_dtype: Optional[torch.dtype] = None,
455
- quant_config: Optional[QuantizationConfig] = None,
456
- ):
485
+ def __init__(self,
486
+ hidden_size: int,
487
+ head_size: int,
488
+ total_num_heads: int,
489
+ total_num_kv_heads: Optional[int] = None,
490
+ bias: bool = True,
491
+ skip_bias_add: bool = False,
492
+ params_dtype: Optional[torch.dtype] = None,
493
+ quant_config: Optional[QuantizationConfig] = None):
494
self.hidden_size = hidden_size
495
self.head_size = head_size
496
self.total_num_heads = total_num_heads
@@ -473,14 +510,19 @@ class QKVParallelLinear(ColumnParallelLinear):
510
input_size = self.hidden_size
511
output_size = (self.num_heads +
512
2 * self.num_kv_heads) * tp_size * self.head_size
476
- output_sizes = [
477
- self.num_heads * tp_size * self.head_size,
478
- self.num_kv_heads * tp_size * self.head_size,
479
- self.num_kv_heads * tp_size * self.head_size
513
+ self.output_sizes = [
514
+ self.num_heads * self.head_size * tp_size, # q_proj
515
+ self.num_kv_heads * self.head_size * tp_size, # k_proj
516
+ self.num_kv_heads * self.head_size * tp_size, # v_proj
517
]
518
482
- super().__init__(input_size, output_size, bias, False, skip_bias_add,
483
- params_dtype, quant_config, output_sizes)
519
+ super().__init__(input_size=input_size,
520
+ output_size=output_size,
521
+ bias=bias,
522
+ gather_output=False,
523
+ skip_bias_add=skip_bias_add,
524
+ params_dtype=params_dtype,
525
+ quant_config=quant_config)
526
527
def weight_loader(self,
528
param: Parameter,
@@ -490,6 +532,26 @@ class QKVParallelLinear(ColumnParallelLinear):
532
output_dim = getattr(param, "output_dim", None)
533
# Special case for AQLM codebooks.
534
is_metadata = getattr(param, "is_metadata", False)
535
+
536
+ param_shard_splitter = getattr(param, "shard_splitter", None)
537
+
538
+ if output_dim is not None and param_shard_splitter is not None:
539
+ raise NotImplementedError(
540
+ "We do not currently support output_dim != None and "
541
+ "shard_splitter != None for a parameter. Please open an issue."
542
+ )
543
+ # If a parameter has defined a shard_splitter to be used for
544
+ # the weight, it should be applied before the weight is
545
+ # loaded/copied to the parameter. The shard_splitter applies
546
+ # logic by using the loaded_shard_id to ensure that the loaded
547
+ # param is loaded to the correct location
548
+ # within the parameter defined by the linear method.
549
+ if loaded_shard_id is None and param_shard_splitter is not None:
550
+ raise NotImplementedError(
551
+ "We do not currently support loaded_shard_id == None and "
552
+ "shard_splitter != None for a parameter. Please open an issue."
553
+ )
554
+
555
# Special case for Fp8 scales.
556
fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
557
None)
@@ -528,6 +590,8 @@ class QKVParallelLinear(ColumnParallelLinear):
590
591
tp_rank = get_tensor_model_parallel_rank()
592
assert loaded_shard_id in ["q", "k", "v"]
593
+
594
+ # If output dim is defined, use the default loading process.
595
if output_dim is not None:
596
if loaded_shard_id == "q":
597
shard_offset = 0
@@ -567,6 +631,12 @@ class QKVParallelLinear(ColumnParallelLinear):
631
shard_index = ["q", "k", "v"].index(loaded_shard_id)
632
param_data = param_data.narrow(0, shard_index * shard_size,
633
shard_size)
634
+ # If a param_shard_splitter is defined by the LinearMethod, use it.
635
+ elif param_shard_splitter is not None:
636
+ logical_widths = getattr(param, "logical_widths", None)
637
+ param_data, loaded_weight = param_shard_splitter(
638
+ param_data, loaded_weight, loaded_shard_id, logical_widths)
639
+
640
# Special case for Fp8 scales.
641
elif fp8_scales_shard_indexer is not None:
642
param_data, loaded_weight = fp8_scales_shard_indexer(
@@ -578,6 +648,13 @@ class QKVParallelLinear(ColumnParallelLinear):
648
"Loading a weight without `output_dim` attribute in "
649
"QKVParallelLinear, assume the weight is the same "
650
"for all partitions.")
651
+
652
+ if len(param_data.shape) == 0:
653
+ param_data = param_data.reshape(1)
654
+
655
+ if len(loaded_weight.shape) == 0:
656
+ loaded_weight = loaded_weight.reshape(1)
657
+
658
assert param_data.shape == loaded_weight.shape
659
param_data.copy_(loaded_weight)
660
@@ -608,17 +685,15 @@ class RowParallelLinear(LinearBase):
685
quant_config: Quantization configure.
686
"""
687
611
- def __init__(
612
- self,
613
- input_size: int,
614
- output_size: int,
615
- bias: bool = True,
616
- input_is_parallel: bool = True,
617
- skip_bias_add: bool = False,
618
- params_dtype: Optional[torch.dtype] = None,
619
- reduce_results: bool = True,
620
- quant_config: Optional[QuantizationConfig] = None,
621
- ):
688
+ def __init__(self,
689
+ input_size: int,
690
+ output_size: int,
691
+ bias: bool = True,
692
+ input_is_parallel: bool = True,
693
+ skip_bias_add: bool = False,
694
+ params_dtype: Optional[torch.dtype] = None,
695
+ reduce_results: bool = True,
696
+ quant_config: Optional[QuantizationConfig] = None):
697
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
698
quant_config)
699
@@ -628,16 +703,15 @@ class RowParallelLinear(LinearBase):
703
# Divide the weight matrix along the last dimension.
704
self.tp_size = get_tensor_model_parallel_world_size()
705
self.input_size_per_partition = divide(input_size, self.tp_size)
631
- # All the linear layer supports quant method.
706
assert self.quant_method is not None
633
- self.quant_method.create_weights(self,
634
- self.input_size_per_partition,
635
- [self.output_size],
636
- self.input_size,
637
- self.output_size,
638
- self.params_dtype,
639
- weight_loader=self.weight_loader)
640
-
707
+ self.quant_method.create_weights(
708
+ layer=self,
709
+ input_size_per_partition=self.input_size_per_partition,
710
+ output_partition_sizes=[self.output_size],
711
+ input_size=self.input_size,
712
+ output_size=self.output_size,
713
+ params_dtype=self.params_dtype,
714
+ weight_loader=self.weight_loader)
715
if not reduce_results and (bias and not skip_bias_add):
716
raise ValueError("When not reduce the results, adding bias to the "
717
"results can lead to incorrect results")
@@ -665,12 +739,16 @@ class RowParallelLinear(LinearBase):
739
start_idx = tp_rank * shard_size
740
loaded_weight = loaded_weight.narrow(input_dim, start_idx,
741
shard_size)
742
+
743
# Special case for Fp8 scales.
744
elif fp8_scales_shard_indexer is not None:
745
param_data, loaded_weight = fp8_scales_shard_indexer(param_data,
746
loaded_weight,
747
shard_id=0)
748
749
+ if fp8_scales_shard_indexer is None and len(loaded_weight.shape) == 0:
750
+ loaded_weight = loaded_weight.reshape(1)
751
+
752
assert param_data.shape == loaded_weight.shape
753
param_data.copy_(loaded_weight)
754
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -4,6 +4,8 @@ from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4
from vllm.model_executor.layers.quantization.awq import AWQConfig
5
from vllm.model_executor.layers.quantization.base_config import (
6
QuantizationConfig)
7
+from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
8
+ CompressedTensorsConfig)
9
from vllm.model_executor.layers.quantization.deepspeedfp import (
10
DeepSpeedFPConfig)
11
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
@@ -27,6 +29,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
29
"gptq_marlin": GPTQMarlinConfig,
30
"gptq": GPTQConfig,
31
"squeezellm": SqueezeLLMConfig,
32
+ "sparseml": CompressedTensorsConfig,
33
}
34
35
vllm/model_executor/layers/quantization/compressed_tensors/__init__.py
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
new
+151
@@ -0,0 +1,151 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+
5
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
6
+from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
7
+ QuantizationConfig)
8
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
9
+ CompressedTensorsScheme, CompressedTensorsW8A8StaticTensor)
10
+
11
+
12
+class CompressedTensorsConfig(QuantizationConfig):
13
+
14
+ def __init__(self, layer_quant_details: Dict[str, Any], ignore: List[str]):
15
+ self.ignore = ignore
16
+ self.layer_quant_details = layer_quant_details
17
+
18
+ def get_linear_method(self) -> "CompressedTensorsLinearMethod":
19
+ return CompressedTensorsLinearMethod(self)
20
+
21
+ def get_scaled_act_names(self) -> List[str]:
22
+ return []
23
+
24
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
25
+ return [torch.float16]
26
+
27
+ # Need to figure it out
28
+ def get_min_capability(self) -> int:
29
+ return 60
30
+
31
+ def get_name(self) -> str:
32
+ return "compressed_tensors"
33
+
34
+ def get_quant_method(
35
+ self, layer: torch.nn.Module
36
+ ) -> Optional["CompressedTensorsLinearMethod"]:
37
+ if isinstance(layer, LinearBase):
38
+ return CompressedTensorsLinearMethod(self)
39
+ return None
40
+
41
+ @classmethod
42
+ def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig":
43
+ layer_quant_details: Dict[str, Any] = dict()
44
+ ignore: List[str] = config.get("ignore", None)
45
+
46
+ for key, quant_config in config["config_groups"].items():
47
+ targets = quant_config.get("targets")
48
+ for target in targets:
49
+ layer_quant_details[target] = {}
50
+ layer_quant_details[target]["weight"] = quant_config.get(
51
+ "weights")
52
+ layer_quant_details[target]["input"] = quant_config.get(
53
+ "input_activations")
54
+
55
+ return cls(layer_quant_details=layer_quant_details, ignore=ignore)
56
+
57
+ @classmethod
58
+ def get_config_filenames(cls) -> List[str]:
59
+ return []
60
+
61
+ def _get_schema(self, weight_quant: Dict, input_quant: Dict):
62
+ # TODO: Refactor as additional cases are supported
63
+
64
+ weight_bit = weight_quant.get("num_bits")
65
+ input_bit = input_quant.get("num_bits")
66
+
67
+ weight_strategy = weight_quant.get("strategy")
68
+ input_strategy = input_quant.get("strategy")
69
+
70
+ weight_symmetric = weight_quant.get("symmetric")
71
+ input_symmetric = input_quant.get("symmetric")
72
+
73
+ is_8_bits = weight_bit == input_bit == 8
74
+ is_tensor = weight_strategy == input_strategy == "tensor"
75
+ is_symmetric = weight_symmetric and input_symmetric
76
+
77
+ if is_8_bits and is_tensor and is_symmetric and \
78
+ torch.cuda.is_available():
79
+ # CompressedTensorsW8A8StaticTensor only supports CUDA path for
80
+ # now.
81
+ return CompressedTensorsW8A8StaticTensor()
82
+ raise NotImplementedError(
83
+ "Scheme not supported. Only CUDA, 8-bit static symmtetric "
84
+ "per tensor quantization is currently supported")
85
+
86
+ def get_scheme(self, layer: torch.nn.Module) -> "CompressedTensorsScheme":
87
+
88
+ # TODO: update with matching function from `compressed_tensors`
89
+ layer_type_name = None
90
+ layer_name_class = type(layer).__name__.lower()
91
+ for target in self.layer_quant_details:
92
+ if target.lower() in layer_name_class:
93
+ layer_type_name = target
94
+ break
95
+ if layer_type_name is None:
96
+ raise ValueError(f"Could not matching target for layer {layer}")
97
+
98
+ layer_quant_details: Dict[str, Any] = self.layer_quant_details.get(
99
+ layer_type_name, None)
100
+ if layer_quant_details is None:
101
+ raise ValueError(
102
+ f"Could not find quantization details for {layer}.")
103
+
104
+ return self._get_schema(weight_quant=layer_quant_details["weight"],
105
+ input_quant=layer_quant_details["input"])
106
+
107
+
108
+class CompressedTensorsLinearMethod(LinearMethodBase):
109
+
110
+ def __init__(self, quantization_config: CompressedTensorsConfig):
111
+ self.quantization_config = quantization_config
112
+
113
+ def create_weights(self, layer: torch.nn.Module,
114
+ input_size_per_partition: int,
115
+ output_partition_sizes: List[int], input_size: int,
116
+ output_size: int, params_dtype: torch.dtype,
117
+ **extra_weight_attrs):
118
+ """
119
+ Use the CompressedTensorsScheme associated with each layer to create
120
+ the necessary parameters for the layer.
121
+ """
122
+ weight_loader = extra_weight_attrs.get("weight_loader")
123
+
124
+ scheme = self.quantization_config.get_scheme(layer=layer)
125
+ scheme.create_weights(
126
+ layer=layer,
127
+ input_size_per_partition=input_size_per_partition,
128
+ output_partition_sizes=output_partition_sizes,
129
+ output_size=output_size,
130
+ params_dtype=params_dtype,
131
+ weight_loader=weight_loader)
132
+
133
+ layer.scheme = scheme
134
+
135
+ def apply(self,
136
+ layer: torch.nn.Module,
137
+ x: torch.Tensor,
138
+ bias: Optional[torch.Tensor] = None):
139
+ """
140
+ Use the output of create_weights and the CompressedTensorsScheme
141
+ associated with the layer to apply the forward pass with the
142
+ layer input.
143
+ """
144
+
145
+ if bias is not None:
146
+ raise ValueError("bias is not supported for this linear method")
147
+
148
+ scheme = layer.scheme
149
+ if scheme is None:
150
+ raise ValueError("A scheme must be defined for each layer")
151
+ return scheme.apply_weights(layer, x)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
new
+5
@@ -0,0 +1,5 @@
1
+from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2
+from .compressed_tensors_unquantized import ( # noqa: F401
3
+ CompressedTensorsUnquantized)
4
+from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
5
+ CompressedTensorsW8A8StaticTensor)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py
new
+33
@@ -0,0 +1,33 @@
1
+from abc import ABC, abstractmethod
2
+
3
+import torch
4
+
5
+__all__ = ["CompressedTensorsScheme"]
6
+
7
+
8
+class CompressedTensorsScheme(ABC):
9
+ """
10
+ Abstract class used to describe the weight creation and forward pass
11
+ of different quantization schemes supported by CompressedTensors.
12
+ """
13
+
14
+ @abstractmethod
15
+ def create_weights(self, *args, **kwargs):
16
+ """
17
+ Weight creation for the particular scheme. Inputs to this function
18
+
19
+ """
20
+ raise NotImplementedError
21
+
22
+ @abstractmethod
23
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
24
+ """
25
+ Run the forward pass for the particular scheme. This is where
26
+ scheme-specific dequant/quant steps/kernels should be applied.
27
+
28
+ :param layer: toch.nn.Module with the registered weights and
29
+ other parameters relevant to the particular scheme.
30
+ :param x: input to the layer
31
+
32
+ """
33
+ raise NotImplementedError
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
new
+39
@@ -0,0 +1,39 @@
1
+from typing import Callable, List
2
+
3
+import torch
4
+import torch.nn.functional as F
5
+from torch.nn import Parameter
6
+
7
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
+ CompressedTensorsScheme)
9
+from vllm.model_executor.utils import set_weight_attrs
10
+
11
+__all__ = ["CompressedTensorsUnquantized"]
12
+
13
+
14
+class CompressedTensorsUnquantized(CompressedTensorsScheme):
15
+ """
16
+ Implements the scheme for all layers which are ignored
17
+ in the CompressedTensors config. The input and loaded weight are used
18
+ in a linear transformation.
19
+ """
20
+
21
+ def create_weights(self, layer: torch.nn.Module,
22
+ output_partition_sizes: List[int],
23
+ input_size_per_partition: int,
24
+ params_dtype: torch.dtype, weight_loader: Callable,
25
+ **kwargs):
26
+
27
+ weight = Parameter(torch.empty(sum(output_partition_sizes),
28
+ input_size_per_partition,
29
+ device="cuda",
30
+ dtype=params_dtype),
31
+ requires_grad=False)
32
+
33
+ set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
34
+ layer.register_parameter("weight", weight)
35
+ set_weight_attrs(weight, {"weight_loader": weight_loader})
36
+
37
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
38
+ weight = layer.weight
39
+ return F.linear(x, weight)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
new
+119
@@ -0,0 +1,119 @@
1
+from typing import Callable, List, Tuple, Union
2
+
3
+import torch
4
+from torch.nn import Parameter
5
+
6
+from vllm import _custom_ops as custom_ops
7
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
+ CompressedTensorsScheme)
9
+from vllm.model_executor.utils import set_weight_attrs
10
+
11
+__all__ = ["CompressedTensorsW8A8StaticTensor"]
12
+
13
+
14
+class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
15
+
16
+ def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
17
+ if isinstance(shard_id, int):
18
+ return shard_id
19
+
20
+ assert isinstance(shard_id, str)
21
+ qkv_idxs = {"q": 0, "k": 1, "v": 2}
22
+ assert shard_id in qkv_idxs
23
+ return qkv_idxs[shard_id]
24
+
25
+ def scales_shard_splitter(
26
+ self, param: torch.Tensor, loaded_weight: torch.Tensor,
27
+ shard_id: Union[str, int],
28
+ logical_widths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
29
+ shard_id = self._shard_id_as_int(shard_id)
30
+ offset = sum(logical_widths[:shard_id])
31
+ size = logical_widths[shard_id]
32
+ # update loaded weight with copies for broadcast.
33
+ loaded_weight = loaded_weight.repeat(size)
34
+ return param[offset:offset + size], loaded_weight
35
+
36
+ def create_weights(self, layer: torch.nn.Module,
37
+ output_partition_sizes: List[int],
38
+ input_size_per_partition: int,
39
+ params_dtype: torch.dtype, weight_loader: Callable,
40
+ **kwargs):
41
+
42
+ # TODO: remove zero_point parameters once the configs given remove them
43
+
44
+ # Note on input/weight scales and zero_points
45
+ #
46
+ # When the scales have a single value, it is required that they be
47
+ # on the CPU for 2 reasons,
48
+ # 1. Performance:
49
+ # When the scales (input_scale/weight_scales) have only a single
50
+ # value, we perform a scalar broadcast of that value during the
51
+ # quant/dequant operations. The "quant" and the "gemm+dequant"
52
+ # kernels accept the Scalar by-value. These tensors are allocated
53
+ # on the CPU in order to avoid the GPU-to-CPU copy when passing
54
+ # by-value.
55
+ #
56
+ # 2. CUDA Graphs:
57
+ # CUDA Graphs don't support GPU-to-CPU copy operations during
58
+ # stream capture.
59
+ #
60
+ # TODO: zero-points are not supported yet. But we expect a similar
61
+ # pattern.
62
+
63
+ is_tensor_partitioned = len(output_partition_sizes) != 1
64
+ weight_scale_dim = sum(
65
+ output_partition_sizes) if is_tensor_partitioned else 1
66
+ weight_scale_device = "cpu" if weight_scale_dim == 1 else "cuda"
67
+
68
+ input_scale = Parameter(torch.empty(1,
69
+ device="cpu",
70
+ dtype=torch.float32),
71
+ requires_grad=False)
72
+ input_zero_point = Parameter(torch.empty(1,
73
+ device="cpu",
74
+ dtype=torch.int8),
75
+ requires_grad=False)
76
+
77
+ weight_scale = Parameter(torch.empty(weight_scale_dim,
78
+ device=weight_scale_device,
79
+ dtype=torch.float32),
80
+ requires_grad=False)
81
+ weight_zero_point = Parameter(torch.empty(1,
82
+ device="cpu",
83
+ dtype=torch.int8),
84
+ requires_grad=False)
85
+
86
+ weight = Parameter(torch.empty(sum(output_partition_sizes),
87
+ input_size_per_partition,
88
+ dtype=torch.int8),
89
+ requires_grad=False)
90
+
91
+ layer.register_parameter("weight", weight)
92
+ set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
93
+
94
+ set_weight_attrs(weight, {"weight_loader": weight_loader})
95
+
96
+ layer.register_parameter("input_scale", input_scale)
97
+ set_weight_attrs(input_scale, {"weight_loader": weight_loader})
98
+ layer.register_parameter("input_zero_point", input_zero_point)
99
+ set_weight_attrs(input_zero_point, {"weight_loader": weight_loader})
100
+ layer.register_parameter("weight_scale", weight_scale)
101
+ set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
102
+ set_weight_attrs(
103
+ weight_scale, {
104
+ "shard_splitter": self.scales_shard_splitter,
105
+ "logical_widths": output_partition_sizes
106
+ })
107
+ layer.register_parameter("weight_zero_point", weight_zero_point)
108
+ set_weight_attrs(weight_zero_point, {"weight_loader": weight_loader})
109
+
110
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
111
+ weight = layer.weight
112
+ weight_scale = layer.weight_scale
113
+ act_scale = layer.input_scale
114
+
115
+ # Input quantize
116
+ x_q = custom_ops.static_scaled_int8_quant(x, act_scale[0].item())
117
+
118
+ return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), act_scale,
119
+ weight_scale, x.dtype)
vllm/model_executor/model_loader/weight_utils.py
+7
@@ -120,6 +120,13 @@ def get_quant_config(model_config: ModelConfig,
120
# Read the quantization config from the HF model config, if available.
121
hf_quant_config = getattr(model_config.hf_config, "quantization_config",
122
None)
123
+ if hf_quant_config is None:
124
+ compression_config = getattr(model_config.hf_config,
125
+ "compression_config", None)
126
+ if compression_config is not None:
127
+ hf_quant_config = compression_config.get("quantization_config",
128
+ None)
129
+
130
if hf_quant_config is not None:
131
return quant_cls.from_config(hf_quant_config)
132
model_name_or_path = model_config.model
vllm/model_executor/models/llama.py
+14
-11
@@ -62,11 +62,12 @@ class LlamaMLP(nn.Module):
62
) -> None:
63
super().__init__()
64
self.gate_up_proj = MergedColumnParallelLinear(
65
- hidden_size, [intermediate_size] * 2,
65
+ input_size=hidden_size,
66
+ output_sizes=[intermediate_size] * 2,
67
bias=bias,
68
quant_config=quant_config)
68
- self.down_proj = RowParallelLinear(intermediate_size,
69
- hidden_size,
69
+ self.down_proj = RowParallelLinear(input_size=intermediate_size,
70
+ output_size=hidden_size,
71
bias=bias,
72
quant_config=quant_config)
73
if hidden_act != "silu":
@@ -120,16 +121,16 @@ class LlamaAttention(nn.Module):
121
self.max_position_embeddings = max_position_embeddings
122
123
self.qkv_proj = QKVParallelLinear(
123
- hidden_size,
124
- self.head_dim,
125
- self.total_num_heads,
126
- self.total_num_kv_heads,
124
+ hidden_size=hidden_size,
125
+ head_size=self.head_dim,
126
+ total_num_heads=self.total_num_heads,
127
+ total_num_kv_heads=self.total_num_kv_heads,
128
bias=bias,
129
quant_config=quant_config,
130
)
131
self.o_proj = RowParallelLinear(
131
- self.total_num_heads * self.head_dim,
132
- hidden_size,
132
+ input_size=self.total_num_heads * self.head_dim,
133
+ output_size=hidden_size,
134
bias=bias,
135
quant_config=quant_config,
136
)
@@ -263,8 +264,10 @@ class LlamaModel(nn.Module):
264
org_num_embeddings=config.vocab_size,
265
)
266
self.layers = nn.ModuleList([
266
- LlamaDecoderLayer(config, cache_config, quant_config)
267
- for _ in range(config.num_hidden_layers)
267
+ LlamaDecoderLayer(config=config,
268
+ cache_config=cache_config,
269
+ quant_config=quant_config)
270
+ for idx in range(config.num_hidden_layers)
271
])
272
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
273