@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a1242324c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Initial Activation Quantization Support (#4525)

Co-authored-by: Varun Sundar Rabindranath <varunsundar08@gmail.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Dipika Sikka committed May 23, 2024 at 17:29 UTC a1242324c99ff8b1e29981006dfb504da198c7c3
17 files changed +683 -94
CMakeLists.txt
+1
@@ -167,6 +167,7 @@ set(VLLM_EXT_SRC
167 "csrc/layernorm_kernels.cu"
168 "csrc/quantization/squeezellm/quant_cuda_kernel.cu"
169 "csrc/quantization/gptq/q_gemm.cu"
170 + "csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
171 "csrc/quantization/fp8/common.cu"
172 "csrc/cuda_utils_kernels.cu"
173 "csrc/moe_align_block_size_kernels.cu"
csrc/ops.h
+3
@@ -93,6 +93,9 @@ int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
93
94 #endif
95
96 +void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor& input,
97 + float scale);
98 +
99 void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
100 torch::Tensor lookup_table);
101
csrc/pybind.cpp
+3
@@ -67,6 +67,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
67 "Aligning the number of tokens to be processed by each expert such "
68 "that it is divisible by the block size.");
69
70 + ops.def("static_scaled_int8_quant", &static_scaled_int8_quant,
71 + "Compute int8 quantized tensor for given scaling factor");
72 +
73 // Cache ops
74 pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
75 cache_ops.def("swap_blocks", &swap_blocks,
csrc/quantization/compressed_tensors/int8_quant_kernels.cu new
+59
@@ -0,0 +1,59 @@
1 +#include <ATen/cuda/CUDAContext.h>
2 +#include <torch/extension.h>
3 +#include <cmath>
4 +
5 +#include "../../dispatch_utils.h"
6 +
7 +static inline __device__ int8_t float_to_int8_rn(float x) {
8 +#ifdef USE_ROCM
9 + static const float i8_min =
10 + static_cast<float>(std::numeric_limits<int8_t>::min());
11 + static const float i8_max =
12 + static_cast<float>(std::numeric_limits<int8_t>::max());
13 + // round
14 + float dst = std::nearbyint(x);
15 + // saturate
16 + dst = std::clamp(dst, i8_min, i8_max);
17 + return static_cast<int8_t>(dst);
18 +#else
19 + // CUDA path
20 + uint32_t dst;
21 + asm volatile("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(dst) : "f"(x));
22 + return reinterpret_cast<const int8_t&>(dst);
23 +#endif
24 +}
25 +
26 +namespace vllm {
27 +
28 +template <typename scalar_t, typename scale_type>
29 +__global__ void static_scaled_int8_quant_kernel(
30 + const scalar_t* __restrict__ input, int8_t* __restrict__ out,
31 + scale_type scale, const int hidden_size) {
32 + const int tid = threadIdx.x;
33 + const int token_idx = blockIdx.x;
34 +
35 + for (int i = tid; i < hidden_size; i += blockDim.x) {
36 + out[token_idx * hidden_size + i] =
37 + float_to_int8_rn(((float)input[token_idx * hidden_size + i]) / scale);
38 + }
39 +}
40 +} // namespace vllm
41 +
42 +void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
43 + torch::Tensor& input, // [..., hidden_size]
44 + float scale) {
45 + TORCH_CHECK(input.is_contiguous());
46 + TORCH_CHECK(out.is_contiguous());
47 + int hidden_size = input.size(-1);
48 + int num_tokens = input.numel() / hidden_size;
49 + dim3 grid(num_tokens);
50 + dim3 block(std::min(hidden_size, 1024));
51 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
52 + VLLM_DISPATCH_FLOATING_TYPES(
53 + input.scalar_type(), "static_scaled_int8_quant_kernel", [&] {
54 + vllm::static_scaled_int8_quant_kernel<scalar_t, float>
55 + <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
56 + out.data_ptr<int8_t>(), scale,
57 + hidden_size);
58 + });
59 +}
tests/kernels/test_int8_quant.py new
+31
@@ -0,0 +1,31 @@
1 +import pytest
2 +import torch
3 +
4 +from vllm._C import ops
5 +
6 +DTYPES = [torch.half, torch.bfloat16, torch.float]
7 +HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 8192] # Arbitrary values for testing
8 +NUM_TOKENS = [1, 7, 83, 4096] # Arbitrary values for testing
9 +SEEDS = [0]
10 +SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
11 +
12 +
13 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
14 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
15 +@pytest.mark.parametrize("dtype", DTYPES)
16 +@pytest.mark.parametrize("seed", SEEDS)
17 +@pytest.mark.parametrize("scale", SCALE)
18 +@torch.inference_mode()
19 +def test_quant(num_tokens: int, hidden_size: int, dtype: torch.dtype,
20 + seed: int, scale: float) -> None:
21 + torch.random.manual_seed(seed)
22 + torch.cuda.manual_seed(seed)
23 + x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
24 +
25 + out1 = (x / scale).round().clamp(
26 + torch.iinfo(torch.int8).min,
27 + torch.iinfo(torch.int8).max).to(torch.int8)
28 + out2 = torch.empty_like(x, dtype=torch.int8)
29 + ops.static_scaled_int8_quant(out2, x, scale)
30 + assert torch.allclose(out1, out2,
31 + atol=1) # big atol to account for rounding errors
tests/quantization/test_compressed_tensors.py new
+36
@@ -0,0 +1,36 @@
1 +"""Test model set-up and weight loading for sparseml-quantized models.
2 +
3 +Run `pytest tests/quantization/test_compressed_tensors.py`.
4 +"""
5 +
6 +import torch
7 +
8 +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
9 + CompressedTensorsLinearMethod, CompressedTensorsW8A8StaticTensor)
10 +
11 +
12 +def test_compressed_tensors_w8a8_static_setup(vllm_runner):
13 + model_path = "nm-testing/tinyllama-one-shot-static-quant-test-compressed"
14 + llm = vllm_runner(model_path, quantization="sparseml", enforce_eager=True)
15 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
16 + layer = model.model.layers[0]
17 +
18 + qkv_proj = layer.self_attn.qkv_proj
19 + o_proj = layer.self_attn.o_proj
20 + gate_up_proj = layer.mlp.gate_up_proj
21 + down_proj = layer.mlp.down_proj
22 +
23 + assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
24 + assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
25 + assert isinstance(gate_up_proj.quant_method, CompressedTensorsLinearMethod)
26 + assert isinstance(down_proj.quant_method, CompressedTensorsLinearMethod)
27 +
28 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8StaticTensor)
29 +
30 + assert qkv_proj.weight.dtype is torch.int8
31 + assert o_proj.weight.dtype is torch.int8
32 + assert gate_up_proj.weight.dtype is torch.int8
33 +
34 + assert qkv_proj.weight_scale.shard_splitter is not None
35 + assert qkv_proj.weight_scale.logical_widths is not None
36 + assert qkv_proj.input_scale.dtype is torch.float32
vllm/_custom_ops.py
+18
@@ -251,6 +251,24 @@ def scaled_fp8_quant(
251 return output, scale
252
253
254 +# int8
255 +def static_scaled_int8_quant(input: torch.Tensor,
256 + scale: float) -> torch.Tensor:
257 + """
258 + Quantize the input tensor to int8 and return the quantized tensor.
259 +
260 + Args:
261 + input: The input tensor to be quantized to int8.
262 + scale: Scaling factor for the int8 quantization.
263 +
264 + Returns:
265 + torch.Tensor: Output tensor in int8.
266 + """
267 + q = torch.empty_like(input, dtype=torch.int8)
268 + vllm_ops.static_scaled_int8_quant(q, input, scale)
269 + return q
270 +
271 +
272 # moe
273 def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
274 block_size: int, sorted_token_ids: torch.Tensor,
vllm/model_executor/layers/linear.py
+161 -83
@@ -56,7 +56,6 @@ class LinearMethodBase(QuantizeMethodBase):
56 x: torch.Tensor,
57 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
58 """Apply the weights in layer to the input tensor.
59 -
59 Expects create_weights to have been called before on the layer."""
60 raise NotImplementedError
61
@@ -77,8 +76,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
76 output_partition_sizes: List[int], input_size: int,
77 output_size: int, params_dtype: torch.dtype,
78 **extra_weight_attrs):
80 - output_size_per_partition = sum(output_partition_sizes)
81 - weight = Parameter(torch.empty(output_size_per_partition,
79 + weight = Parameter(torch.empty(sum(output_partition_sizes),
80 input_size_per_partition,
81 dtype=params_dtype),
82 requires_grad=False)
@@ -149,15 +147,13 @@ class ReplicatedLinear(LinearBase):
147 quant_config: Quantization configure.
148 """
149
152 - def __init__(
153 - self,
154 - input_size: int,
155 - output_size: int,
156 - bias: bool = True,
157 - skip_bias_add: bool = False,
158 - params_dtype: Optional[torch.dtype] = None,
159 - quant_config: Optional[QuantizationConfig] = None,
160 - ):
150 + def __init__(self,
151 + input_size: int,
152 + output_size: int,
153 + bias: bool = True,
154 + skip_bias_add: bool = False,
155 + params_dtype: Optional[torch.dtype] = None,
156 + quant_config: Optional[QuantizationConfig] = None):
157 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
158 quant_config)
159
@@ -210,17 +206,15 @@ class ColumnParallelLinear(LinearBase):
206 the list would be size 3.
207 """
208
213 - def __init__(
214 - self,
215 - input_size: int,
216 - output_size: int,
217 - bias: bool = True,
218 - gather_output: bool = False,
219 - skip_bias_add: bool = False,
220 - params_dtype: Optional[torch.dtype] = None,
221 - quant_config: Optional[QuantizationConfig] = None,
222 - output_sizes: Optional[List[int]] = None,
223 - ):
209 + def __init__(self,
210 + input_size: int,
211 + output_size: int,
212 + bias: bool = True,
213 + gather_output: bool = False,
214 + skip_bias_add: bool = False,
215 + params_dtype: Optional[torch.dtype] = None,
216 + quant_config: Optional[QuantizationConfig] = None,
217 + output_sizes: Optional[List[int]] = None):
218 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
219 quant_config)
220
@@ -228,18 +222,26 @@ class ColumnParallelLinear(LinearBase):
222
223 # Divide the weight matrix along the last dimension.
224 tp_size = get_tensor_model_parallel_world_size()
231 - self.output_size_per_partition = divide(output_size, tp_size)
225 + assert self.quant_method is not None
226 + self.output_size_per_partition = divide(self.output_size, tp_size)
227 + self.output_partition_sizes = [self.output_size_per_partition]
228 + # If QKV or MergedColumn, use output size of each partition.
229 + if hasattr(self, "output_sizes"):
230 + self.output_partition_sizes = [
231 + divide(output_size, tp_size)
232 + for output_size in self.output_sizes
233 + ]
234 +
235 if output_sizes is None:
236 output_sizes = [output_size]
234 - # All the linear layer supports quant method.
235 - assert self.quant_method is not None
236 - self.quant_method.create_weights(self,
237 - self.input_size,
238 - [x // tp_size for x in output_sizes],
239 - self.input_size,
240 - self.output_size,
241 - self.params_dtype,
242 - weight_loader=self.weight_loader)
237 + self.quant_method.create_weights(
238 + layer=self,
239 + input_size_per_partition=self.input_size,
240 + output_partition_sizes=self.output_partition_sizes,
241 + input_size=self.input_size,
242 + output_size=self.output_size,
243 + params_dtype=self.params_dtype,
244 + weight_loader=self.weight_loader)
245 if bias:
246 self.bias = Parameter(
247 torch.empty(self.output_size_per_partition,
@@ -317,22 +319,24 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
319 quant_config: Quantization configure.
320 """
321
320 - def __init__(
321 - self,
322 - input_size: int,
323 - output_sizes: List[int],
324 - bias: bool = True,
325 - gather_output: bool = False,
326 - skip_bias_add: bool = False,
327 - params_dtype: Optional[torch.dtype] = None,
328 - quant_config: Optional[QuantizationConfig] = None,
329 - ):
322 + def __init__(self,
323 + input_size: int,
324 + output_sizes: List[int],
325 + bias: bool = True,
326 + gather_output: bool = False,
327 + skip_bias_add: bool = False,
328 + params_dtype: Optional[torch.dtype] = None,
329 + quant_config: Optional[QuantizationConfig] = None):
330 self.output_sizes = output_sizes
331 tp_size = get_tensor_model_parallel_world_size()
332 assert all(output_size % tp_size == 0 for output_size in output_sizes)
333 - super().__init__(input_size, sum(output_sizes), bias, gather_output,
334 - skip_bias_add, params_dtype, quant_config,
335 - self.output_sizes)
333 + super().__init__(input_size=input_size,
334 + output_size=sum(output_sizes),
335 + bias=bias,
336 + gather_output=gather_output,
337 + skip_bias_add=skip_bias_add,
338 + params_dtype=params_dtype,
339 + quant_config=quant_config)
340
341 def weight_loader(self,
342 param: Parameter,
@@ -343,6 +347,26 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
347 output_dim = getattr(param, "output_dim", None)
348 # Special case for AQLM codebooks.
349 is_metadata = getattr(param, "is_metadata", False)
350 +
351 + param_shard_splitter = getattr(param, "shard_splitter", None)
352 +
353 + if output_dim is not None and param_shard_splitter is not None:
354 + raise NotImplementedError(
355 + "We do not currently support output_dim != None and "
356 + "shard_splitter != None for a parameter. Please open an issue."
357 + )
358 + # If a parameter has defined a shard_splitter to be used for
359 + # the weight, it should be applied before the weight is
360 + # loaded/copied to the parameter. The shard_splitter applies
361 + # logic by using the loaded_shard_id to ensure that the loaded
362 + # param is loaded to the correct location
363 + # within the parameter defined by the linear method.
364 + if loaded_shard_id is None and param_shard_splitter is not None:
365 + raise NotImplementedError(
366 + "We do not currently support loaded_shard_id == None and "
367 + "shard_splitter != None for a parameter. Please open an issue."
368 + )
369 +
370 # Special case for Fp8 scales.
371 fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
372 None)
@@ -403,6 +427,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
427 shard_size = loaded_weight.shape[0]
428 shard_offset = loaded_shard_id * shard_size
429 param_data = param_data.narrow(0, shard_offset, shard_size)
430 +
431 + # If a param_shard_splitter is defined by the LinearMethod, use it.
432 + elif param_shard_splitter is not None:
433 + logical_widths = getattr(param, "logical_widths", None)
434 + param_data, loaded_weight = param_shard_splitter(
435 + param_data, loaded_weight, loaded_shard_id, logical_widths)
436 +
437 # Special case for Fp8 scales.
438 elif fp8_scales_shard_indexer is not None:
439 param_data, loaded_weight = fp8_scales_shard_indexer(
@@ -415,6 +446,14 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
446 "Loading a weight without `output_dim` attribute in "
447 "MergedColumnParallelLinear, assume the weight is "
448 "the same for all partitions.")
449 +
450 + if fp8_scales_shard_indexer is None:
451 + if len(param_data.shape) == 0:
452 + param_data = param_data.reshape(1)
453 +
454 + if len(loaded_weight.shape) == 0:
455 + loaded_weight = loaded_weight.reshape(1)
456 +
457 assert param_data.shape == loaded_weight.shape
458 param_data.copy_(loaded_weight)
459
@@ -443,17 +482,15 @@ class QKVParallelLinear(ColumnParallelLinear):
482 quant_config: Quantization configure.
483 """
484
446 - def __init__(
447 - self,
448 - hidden_size: int,
449 - head_size: int,
450 - total_num_heads: int,
451 - total_num_kv_heads: Optional[int] = None,
452 - bias: bool = True,
453 - skip_bias_add: bool = False,
454 - params_dtype: Optional[torch.dtype] = None,
455 - quant_config: Optional[QuantizationConfig] = None,
456 - ):
485 + def __init__(self,
486 + hidden_size: int,
487 + head_size: int,
488 + total_num_heads: int,
489 + total_num_kv_heads: Optional[int] = None,
490 + bias: bool = True,
491 + skip_bias_add: bool = False,
492 + params_dtype: Optional[torch.dtype] = None,
493 + quant_config: Optional[QuantizationConfig] = None):
494 self.hidden_size = hidden_size
495 self.head_size = head_size
496 self.total_num_heads = total_num_heads
@@ -473,14 +510,19 @@ class QKVParallelLinear(ColumnParallelLinear):
510 input_size = self.hidden_size
511 output_size = (self.num_heads +
512 2 * self.num_kv_heads) * tp_size * self.head_size
476 - output_sizes = [
477 - self.num_heads * tp_size * self.head_size,
478 - self.num_kv_heads * tp_size * self.head_size,
479 - self.num_kv_heads * tp_size * self.head_size
513 + self.output_sizes = [
514 + self.num_heads * self.head_size * tp_size, # q_proj
515 + self.num_kv_heads * self.head_size * tp_size, # k_proj
516 + self.num_kv_heads * self.head_size * tp_size, # v_proj
517 ]
518
482 - super().__init__(input_size, output_size, bias, False, skip_bias_add,
483 - params_dtype, quant_config, output_sizes)
519 + super().__init__(input_size=input_size,
520 + output_size=output_size,
521 + bias=bias,
522 + gather_output=False,
523 + skip_bias_add=skip_bias_add,
524 + params_dtype=params_dtype,
525 + quant_config=quant_config)
526
527 def weight_loader(self,
528 param: Parameter,
@@ -490,6 +532,26 @@ class QKVParallelLinear(ColumnParallelLinear):
532 output_dim = getattr(param, "output_dim", None)
533 # Special case for AQLM codebooks.
534 is_metadata = getattr(param, "is_metadata", False)
535 +
536 + param_shard_splitter = getattr(param, "shard_splitter", None)
537 +
538 + if output_dim is not None and param_shard_splitter is not None:
539 + raise NotImplementedError(
540 + "We do not currently support output_dim != None and "
541 + "shard_splitter != None for a parameter. Please open an issue."
542 + )
543 + # If a parameter has defined a shard_splitter to be used for
544 + # the weight, it should be applied before the weight is
545 + # loaded/copied to the parameter. The shard_splitter applies
546 + # logic by using the loaded_shard_id to ensure that the loaded
547 + # param is loaded to the correct location
548 + # within the parameter defined by the linear method.
549 + if loaded_shard_id is None and param_shard_splitter is not None:
550 + raise NotImplementedError(
551 + "We do not currently support loaded_shard_id == None and "
552 + "shard_splitter != None for a parameter. Please open an issue."
553 + )
554 +
555 # Special case for Fp8 scales.
556 fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
557 None)
@@ -528,6 +590,8 @@ class QKVParallelLinear(ColumnParallelLinear):
590
591 tp_rank = get_tensor_model_parallel_rank()
592 assert loaded_shard_id in ["q", "k", "v"]
593 +
594 + # If output dim is defined, use the default loading process.
595 if output_dim is not None:
596 if loaded_shard_id == "q":
597 shard_offset = 0
@@ -567,6 +631,12 @@ class QKVParallelLinear(ColumnParallelLinear):
631 shard_index = ["q", "k", "v"].index(loaded_shard_id)
632 param_data = param_data.narrow(0, shard_index * shard_size,
633 shard_size)
634 + # If a param_shard_splitter is defined by the LinearMethod, use it.
635 + elif param_shard_splitter is not None:
636 + logical_widths = getattr(param, "logical_widths", None)
637 + param_data, loaded_weight = param_shard_splitter(
638 + param_data, loaded_weight, loaded_shard_id, logical_widths)
639 +
640 # Special case for Fp8 scales.
641 elif fp8_scales_shard_indexer is not None:
642 param_data, loaded_weight = fp8_scales_shard_indexer(
@@ -578,6 +648,13 @@ class QKVParallelLinear(ColumnParallelLinear):
648 "Loading a weight without `output_dim` attribute in "
649 "QKVParallelLinear, assume the weight is the same "
650 "for all partitions.")
651 +
652 + if len(param_data.shape) == 0:
653 + param_data = param_data.reshape(1)
654 +
655 + if len(loaded_weight.shape) == 0:
656 + loaded_weight = loaded_weight.reshape(1)
657 +
658 assert param_data.shape == loaded_weight.shape
659 param_data.copy_(loaded_weight)
660
@@ -608,17 +685,15 @@ class RowParallelLinear(LinearBase):
685 quant_config: Quantization configure.
686 """
687
611 - def __init__(
612 - self,
613 - input_size: int,
614 - output_size: int,
615 - bias: bool = True,
616 - input_is_parallel: bool = True,
617 - skip_bias_add: bool = False,
618 - params_dtype: Optional[torch.dtype] = None,
619 - reduce_results: bool = True,
620 - quant_config: Optional[QuantizationConfig] = None,
621 - ):
688 + def __init__(self,
689 + input_size: int,
690 + output_size: int,
691 + bias: bool = True,
692 + input_is_parallel: bool = True,
693 + skip_bias_add: bool = False,
694 + params_dtype: Optional[torch.dtype] = None,
695 + reduce_results: bool = True,
696 + quant_config: Optional[QuantizationConfig] = None):
697 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
698 quant_config)
699
@@ -628,16 +703,15 @@ class RowParallelLinear(LinearBase):
703 # Divide the weight matrix along the last dimension.
704 self.tp_size = get_tensor_model_parallel_world_size()
705 self.input_size_per_partition = divide(input_size, self.tp_size)
631 - # All the linear layer supports quant method.
706 assert self.quant_method is not None
633 - self.quant_method.create_weights(self,
634 - self.input_size_per_partition,
635 - [self.output_size],
636 - self.input_size,
637 - self.output_size,
638 - self.params_dtype,
639 - weight_loader=self.weight_loader)
640 -
707 + self.quant_method.create_weights(
708 + layer=self,
709 + input_size_per_partition=self.input_size_per_partition,
710 + output_partition_sizes=[self.output_size],
711 + input_size=self.input_size,
712 + output_size=self.output_size,
713 + params_dtype=self.params_dtype,
714 + weight_loader=self.weight_loader)
715 if not reduce_results and (bias and not skip_bias_add):
716 raise ValueError("When not reduce the results, adding bias to the "
717 "results can lead to incorrect results")
@@ -665,12 +739,16 @@ class RowParallelLinear(LinearBase):
739 start_idx = tp_rank * shard_size
740 loaded_weight = loaded_weight.narrow(input_dim, start_idx,
741 shard_size)
742 +
743 # Special case for Fp8 scales.
744 elif fp8_scales_shard_indexer is not None:
745 param_data, loaded_weight = fp8_scales_shard_indexer(param_data,
746 loaded_weight,
747 shard_id=0)
748
749 + if fp8_scales_shard_indexer is None and len(loaded_weight.shape) == 0:
750 + loaded_weight = loaded_weight.reshape(1)
751 +
752 assert param_data.shape == loaded_weight.shape
753 param_data.copy_(loaded_weight)
754
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -4,6 +4,8 @@ from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4 from vllm.model_executor.layers.quantization.awq import AWQConfig
5 from vllm.model_executor.layers.quantization.base_config import (
6 QuantizationConfig)
7 +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
8 + CompressedTensorsConfig)
9 from vllm.model_executor.layers.quantization.deepspeedfp import (
10 DeepSpeedFPConfig)
11 from vllm.model_executor.layers.quantization.fp8 import Fp8Config
@@ -27,6 +29,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
29 "gptq_marlin": GPTQMarlinConfig,
30 "gptq": GPTQConfig,
31 "squeezellm": SqueezeLLMConfig,
32 + "sparseml": CompressedTensorsConfig,
33 }
34
35
vllm/model_executor/layers/quantization/compressed_tensors/__init__.py
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py new
+151
@@ -0,0 +1,151 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +
5 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
6 +from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
7 + QuantizationConfig)
8 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
9 + CompressedTensorsScheme, CompressedTensorsW8A8StaticTensor)
10 +
11 +
12 +class CompressedTensorsConfig(QuantizationConfig):
13 +
14 + def __init__(self, layer_quant_details: Dict[str, Any], ignore: List[str]):
15 + self.ignore = ignore
16 + self.layer_quant_details = layer_quant_details
17 +
18 + def get_linear_method(self) -> "CompressedTensorsLinearMethod":
19 + return CompressedTensorsLinearMethod(self)
20 +
21 + def get_scaled_act_names(self) -> List[str]:
22 + return []
23 +
24 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
25 + return [torch.float16]
26 +
27 + # Need to figure it out
28 + def get_min_capability(self) -> int:
29 + return 60
30 +
31 + def get_name(self) -> str:
32 + return "compressed_tensors"
33 +
34 + def get_quant_method(
35 + self, layer: torch.nn.Module
36 + ) -> Optional["CompressedTensorsLinearMethod"]:
37 + if isinstance(layer, LinearBase):
38 + return CompressedTensorsLinearMethod(self)
39 + return None
40 +
41 + @classmethod
42 + def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig":
43 + layer_quant_details: Dict[str, Any] = dict()
44 + ignore: List[str] = config.get("ignore", None)
45 +
46 + for key, quant_config in config["config_groups"].items():
47 + targets = quant_config.get("targets")
48 + for target in targets:
49 + layer_quant_details[target] = {}
50 + layer_quant_details[target]["weight"] = quant_config.get(
51 + "weights")
52 + layer_quant_details[target]["input"] = quant_config.get(
53 + "input_activations")
54 +
55 + return cls(layer_quant_details=layer_quant_details, ignore=ignore)
56 +
57 + @classmethod
58 + def get_config_filenames(cls) -> List[str]:
59 + return []
60 +
61 + def _get_schema(self, weight_quant: Dict, input_quant: Dict):
62 + # TODO: Refactor as additional cases are supported
63 +
64 + weight_bit = weight_quant.get("num_bits")
65 + input_bit = input_quant.get("num_bits")
66 +
67 + weight_strategy = weight_quant.get("strategy")
68 + input_strategy = input_quant.get("strategy")
69 +
70 + weight_symmetric = weight_quant.get("symmetric")
71 + input_symmetric = input_quant.get("symmetric")
72 +
73 + is_8_bits = weight_bit == input_bit == 8
74 + is_tensor = weight_strategy == input_strategy == "tensor"
75 + is_symmetric = weight_symmetric and input_symmetric
76 +
77 + if is_8_bits and is_tensor and is_symmetric and \
78 + torch.cuda.is_available():
79 + # CompressedTensorsW8A8StaticTensor only supports CUDA path for
80 + # now.
81 + return CompressedTensorsW8A8StaticTensor()
82 + raise NotImplementedError(
83 + "Scheme not supported. Only CUDA, 8-bit static symmtetric "
84 + "per tensor quantization is currently supported")
85 +
86 + def get_scheme(self, layer: torch.nn.Module) -> "CompressedTensorsScheme":
87 +
88 + # TODO: update with matching function from `compressed_tensors`
89 + layer_type_name = None
90 + layer_name_class = type(layer).__name__.lower()
91 + for target in self.layer_quant_details:
92 + if target.lower() in layer_name_class:
93 + layer_type_name = target
94 + break
95 + if layer_type_name is None:
96 + raise ValueError(f"Could not matching target for layer {layer}")
97 +
98 + layer_quant_details: Dict[str, Any] = self.layer_quant_details.get(
99 + layer_type_name, None)
100 + if layer_quant_details is None:
101 + raise ValueError(
102 + f"Could not find quantization details for {layer}.")
103 +
104 + return self._get_schema(weight_quant=layer_quant_details["weight"],
105 + input_quant=layer_quant_details["input"])
106 +
107 +
108 +class CompressedTensorsLinearMethod(LinearMethodBase):
109 +
110 + def __init__(self, quantization_config: CompressedTensorsConfig):
111 + self.quantization_config = quantization_config
112 +
113 + def create_weights(self, layer: torch.nn.Module,
114 + input_size_per_partition: int,
115 + output_partition_sizes: List[int], input_size: int,
116 + output_size: int, params_dtype: torch.dtype,
117 + **extra_weight_attrs):
118 + """
119 + Use the CompressedTensorsScheme associated with each layer to create
120 + the necessary parameters for the layer.
121 + """
122 + weight_loader = extra_weight_attrs.get("weight_loader")
123 +
124 + scheme = self.quantization_config.get_scheme(layer=layer)
125 + scheme.create_weights(
126 + layer=layer,
127 + input_size_per_partition=input_size_per_partition,
128 + output_partition_sizes=output_partition_sizes,
129 + output_size=output_size,
130 + params_dtype=params_dtype,
131 + weight_loader=weight_loader)
132 +
133 + layer.scheme = scheme
134 +
135 + def apply(self,
136 + layer: torch.nn.Module,
137 + x: torch.Tensor,
138 + bias: Optional[torch.Tensor] = None):
139 + """
140 + Use the output of create_weights and the CompressedTensorsScheme
141 + associated with the layer to apply the forward pass with the
142 + layer input.
143 + """
144 +
145 + if bias is not None:
146 + raise ValueError("bias is not supported for this linear method")
147 +
148 + scheme = layer.scheme
149 + if scheme is None:
150 + raise ValueError("A scheme must be defined for each layer")
151 + return scheme.apply_weights(layer, x)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py new
+5
@@ -0,0 +1,5 @@
1 +from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2 +from .compressed_tensors_unquantized import ( # noqa: F401
3 + CompressedTensorsUnquantized)
4 +from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
5 + CompressedTensorsW8A8StaticTensor)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py new
+33
@@ -0,0 +1,33 @@
1 +from abc import ABC, abstractmethod
2 +
3 +import torch
4 +
5 +__all__ = ["CompressedTensorsScheme"]
6 +
7 +
8 +class CompressedTensorsScheme(ABC):
9 + """
10 + Abstract class used to describe the weight creation and forward pass
11 + of different quantization schemes supported by CompressedTensors.
12 + """
13 +
14 + @abstractmethod
15 + def create_weights(self, *args, **kwargs):
16 + """
17 + Weight creation for the particular scheme. Inputs to this function
18 +
19 + """
20 + raise NotImplementedError
21 +
22 + @abstractmethod
23 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
24 + """
25 + Run the forward pass for the particular scheme. This is where
26 + scheme-specific dequant/quant steps/kernels should be applied.
27 +
28 + :param layer: toch.nn.Module with the registered weights and
29 + other parameters relevant to the particular scheme.
30 + :param x: input to the layer
31 +
32 + """
33 + raise NotImplementedError
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py new
+39
@@ -0,0 +1,39 @@
1 +from typing import Callable, List
2 +
3 +import torch
4 +import torch.nn.functional as F
5 +from torch.nn import Parameter
6 +
7 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 + CompressedTensorsScheme)
9 +from vllm.model_executor.utils import set_weight_attrs
10 +
11 +__all__ = ["CompressedTensorsUnquantized"]
12 +
13 +
14 +class CompressedTensorsUnquantized(CompressedTensorsScheme):
15 + """
16 + Implements the scheme for all layers which are ignored
17 + in the CompressedTensors config. The input and loaded weight are used
18 + in a linear transformation.
19 + """
20 +
21 + def create_weights(self, layer: torch.nn.Module,
22 + output_partition_sizes: List[int],
23 + input_size_per_partition: int,
24 + params_dtype: torch.dtype, weight_loader: Callable,
25 + **kwargs):
26 +
27 + weight = Parameter(torch.empty(sum(output_partition_sizes),
28 + input_size_per_partition,
29 + device="cuda",
30 + dtype=params_dtype),
31 + requires_grad=False)
32 +
33 + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
34 + layer.register_parameter("weight", weight)
35 + set_weight_attrs(weight, {"weight_loader": weight_loader})
36 +
37 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
38 + weight = layer.weight
39 + return F.linear(x, weight)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py new
+119
@@ -0,0 +1,119 @@
1 +from typing import Callable, List, Tuple, Union
2 +
3 +import torch
4 +from torch.nn import Parameter
5 +
6 +from vllm import _custom_ops as custom_ops
7 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 + CompressedTensorsScheme)
9 +from vllm.model_executor.utils import set_weight_attrs
10 +
11 +__all__ = ["CompressedTensorsW8A8StaticTensor"]
12 +
13 +
14 +class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
15 +
16 + def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
17 + if isinstance(shard_id, int):
18 + return shard_id
19 +
20 + assert isinstance(shard_id, str)
21 + qkv_idxs = {"q": 0, "k": 1, "v": 2}
22 + assert shard_id in qkv_idxs
23 + return qkv_idxs[shard_id]
24 +
25 + def scales_shard_splitter(
26 + self, param: torch.Tensor, loaded_weight: torch.Tensor,
27 + shard_id: Union[str, int],
28 + logical_widths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
29 + shard_id = self._shard_id_as_int(shard_id)
30 + offset = sum(logical_widths[:shard_id])
31 + size = logical_widths[shard_id]
32 + # update loaded weight with copies for broadcast.
33 + loaded_weight = loaded_weight.repeat(size)
34 + return param[offset:offset + size], loaded_weight
35 +
36 + def create_weights(self, layer: torch.nn.Module,
37 + output_partition_sizes: List[int],
38 + input_size_per_partition: int,
39 + params_dtype: torch.dtype, weight_loader: Callable,
40 + **kwargs):
41 +
42 + # TODO: remove zero_point parameters once the configs given remove them
43 +
44 + # Note on input/weight scales and zero_points
45 + #
46 + # When the scales have a single value, it is required that they be
47 + # on the CPU for 2 reasons,
48 + # 1. Performance:
49 + # When the scales (input_scale/weight_scales) have only a single
50 + # value, we perform a scalar broadcast of that value during the
51 + # quant/dequant operations. The "quant" and the "gemm+dequant"
52 + # kernels accept the Scalar by-value. These tensors are allocated
53 + # on the CPU in order to avoid the GPU-to-CPU copy when passing
54 + # by-value.
55 + #
56 + # 2. CUDA Graphs:
57 + # CUDA Graphs don't support GPU-to-CPU copy operations during
58 + # stream capture.
59 + #
60 + # TODO: zero-points are not supported yet. But we expect a similar
61 + # pattern.
62 +
63 + is_tensor_partitioned = len(output_partition_sizes) != 1
64 + weight_scale_dim = sum(
65 + output_partition_sizes) if is_tensor_partitioned else 1
66 + weight_scale_device = "cpu" if weight_scale_dim == 1 else "cuda"
67 +
68 + input_scale = Parameter(torch.empty(1,
69 + device="cpu",
70 + dtype=torch.float32),
71 + requires_grad=False)
72 + input_zero_point = Parameter(torch.empty(1,
73 + device="cpu",
74 + dtype=torch.int8),
75 + requires_grad=False)
76 +
77 + weight_scale = Parameter(torch.empty(weight_scale_dim,
78 + device=weight_scale_device,
79 + dtype=torch.float32),
80 + requires_grad=False)
81 + weight_zero_point = Parameter(torch.empty(1,
82 + device="cpu",
83 + dtype=torch.int8),
84 + requires_grad=False)
85 +
86 + weight = Parameter(torch.empty(sum(output_partition_sizes),
87 + input_size_per_partition,
88 + dtype=torch.int8),
89 + requires_grad=False)
90 +
91 + layer.register_parameter("weight", weight)
92 + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
93 +
94 + set_weight_attrs(weight, {"weight_loader": weight_loader})
95 +
96 + layer.register_parameter("input_scale", input_scale)
97 + set_weight_attrs(input_scale, {"weight_loader": weight_loader})
98 + layer.register_parameter("input_zero_point", input_zero_point)
99 + set_weight_attrs(input_zero_point, {"weight_loader": weight_loader})
100 + layer.register_parameter("weight_scale", weight_scale)
101 + set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
102 + set_weight_attrs(
103 + weight_scale, {
104 + "shard_splitter": self.scales_shard_splitter,
105 + "logical_widths": output_partition_sizes
106 + })
107 + layer.register_parameter("weight_zero_point", weight_zero_point)
108 + set_weight_attrs(weight_zero_point, {"weight_loader": weight_loader})
109 +
110 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
111 + weight = layer.weight
112 + weight_scale = layer.weight_scale
113 + act_scale = layer.input_scale
114 +
115 + # Input quantize
116 + x_q = custom_ops.static_scaled_int8_quant(x, act_scale[0].item())
117 +
118 + return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), act_scale,
119 + weight_scale, x.dtype)
vllm/model_executor/model_loader/weight_utils.py
+7
@@ -120,6 +120,13 @@ def get_quant_config(model_config: ModelConfig,
120 # Read the quantization config from the HF model config, if available.
121 hf_quant_config = getattr(model_config.hf_config, "quantization_config",
122 None)
123 + if hf_quant_config is None:
124 + compression_config = getattr(model_config.hf_config,
125 + "compression_config", None)
126 + if compression_config is not None:
127 + hf_quant_config = compression_config.get("quantization_config",
128 + None)
129 +
130 if hf_quant_config is not None:
131 return quant_cls.from_config(hf_quant_config)
132 model_name_or_path = model_config.model
vllm/model_executor/models/llama.py
+14 -11
@@ -62,11 +62,12 @@ class LlamaMLP(nn.Module):
62 ) -> None:
63 super().__init__()
64 self.gate_up_proj = MergedColumnParallelLinear(
65 - hidden_size, [intermediate_size] * 2,
65 + input_size=hidden_size,
66 + output_sizes=[intermediate_size] * 2,
67 bias=bias,
68 quant_config=quant_config)
68 - self.down_proj = RowParallelLinear(intermediate_size,
69 - hidden_size,
69 + self.down_proj = RowParallelLinear(input_size=intermediate_size,
70 + output_size=hidden_size,
71 bias=bias,
72 quant_config=quant_config)
73 if hidden_act != "silu":
@@ -120,16 +121,16 @@ class LlamaAttention(nn.Module):
121 self.max_position_embeddings = max_position_embeddings
122
123 self.qkv_proj = QKVParallelLinear(
123 - hidden_size,
124 - self.head_dim,
125 - self.total_num_heads,
126 - self.total_num_kv_heads,
124 + hidden_size=hidden_size,
125 + head_size=self.head_dim,
126 + total_num_heads=self.total_num_heads,
127 + total_num_kv_heads=self.total_num_kv_heads,
128 bias=bias,
129 quant_config=quant_config,
130 )
131 self.o_proj = RowParallelLinear(
131 - self.total_num_heads * self.head_dim,
132 - hidden_size,
132 + input_size=self.total_num_heads * self.head_dim,
133 + output_size=hidden_size,
134 bias=bias,
135 quant_config=quant_config,
136 )
@@ -263,8 +264,10 @@ class LlamaModel(nn.Module):
264 org_num_embeddings=config.vocab_size,
265 )
266 self.layers = nn.ModuleList([
266 - LlamaDecoderLayer(config, cache_config, quant_config)
267 - for _ in range(config.num_hidden_layers)
267 + LlamaDecoderLayer(config=config,
268 + cache_config=cache_config,
269 + quant_config=quant_config)
270 + for idx in range(config.num_hidden_layers)
271 ])
272 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
273