@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ca3ea51bde
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Dynamic Per-Token Activation Quantization (#5037)

Co-authored-by: Varun Sundar Rabindranath <varunsundar08@gmail.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Dipika Sikka committed Jun 7, 2024 at 12:36 UTC ca3ea51bde6c22d0afb3aa0a3fdba6d568095a0a
12 files changed +440 -76
csrc/ops.h
+3
@@ -97,6 +97,9 @@ int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
97 void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
98 torch::Tensor const& scale);
99
100 +void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
101 + torch::Tensor& scales);
102 +
103 void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
104 torch::Tensor lookup_table);
105
csrc/pybind.cpp
+3
@@ -70,6 +70,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
70 ops.def("static_scaled_int8_quant", &static_scaled_int8_quant,
71 "Compute int8 quantized tensor for given scaling factor");
72
73 + ops.def("dynamic_scaled_int8_quant", &dynamic_scaled_int8_quant,
74 + "Compute int8 quantized tensor and scaling factor");
75 +
76 // Cache ops
77 pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
78 cache_ops.def("swap_blocks", &swap_blocks,
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+64 -11
@@ -3,6 +3,7 @@
3 #include <cmath>
4
5 #include "../../dispatch_utils.h"
6 +#include "../../reduction_utils.cuh"
7
8 static inline __device__ int8_t float_to_int8_rn(float x) {
9 #ifdef USE_ROCM
@@ -27,17 +28,48 @@ namespace vllm {
28
29 template <typename scalar_t, typename scale_type>
30 __global__ void static_scaled_int8_quant_kernel(
30 - const scalar_t* __restrict__ input, int8_t* __restrict__ out,
31 - const scale_type* scale_ptr, const int hidden_size) {
32 - const int tid = threadIdx.x;
33 - const int token_idx = blockIdx.x;
34 - scale_type scale = *scale_ptr;
31 + scalar_t const* __restrict__ input, int8_t* __restrict__ out,
32 + scale_type const* scale_ptr, const int hidden_size) {
33 + int const tid = threadIdx.x;
34 + int const token_idx = blockIdx.x;
35 + scale_type const scale = *scale_ptr;
36
37 for (int i = tid; i < hidden_size; i += blockDim.x) {
37 - out[token_idx * hidden_size + i] =
38 - float_to_int8_rn(((float)input[token_idx * hidden_size + i]) / scale);
38 + out[token_idx * hidden_size + i] = float_to_int8_rn(
39 + static_cast<float>(input[token_idx * hidden_size + i]) / scale);
40 }
41 }
42 +
43 +template <typename scalar_t, typename scale_type>
44 +__global__ void dynamic_scaled_int8_quant_kernel(
45 + scalar_t const* __restrict__ input, int8_t* __restrict__ out,
46 + scale_type* scale, const int hidden_size) {
47 + int const tid = threadIdx.x;
48 + int const token_idx = blockIdx.x;
49 + float absmax_val = 0.0f;
50 + float const zero = 0.0f;
51 +
52 + for (int i = tid; i < hidden_size; i += blockDim.x) {
53 + float val = static_cast<float>(input[token_idx * hidden_size + i]);
54 + val = val > zero ? val : -val;
55 + absmax_val = val > absmax_val ? val : absmax_val;
56 + }
57 +
58 + float const block_absmax_val_maybe = blockReduceMax(absmax_val);
59 + __shared__ float block_absmax_val;
60 + if (tid == 0) {
61 + block_absmax_val = block_absmax_val_maybe;
62 + scale[token_idx] = block_absmax_val / 127.0f;
63 + }
64 + __syncthreads();
65 +
66 + float const tmp_scale = 127.0f / block_absmax_val;
67 + for (int i = tid; i < hidden_size; i += blockDim.x) {
68 + out[token_idx * hidden_size + i] = float_to_int8_rn(
69 + static_cast<float>(input[token_idx * hidden_size + i]) * tmp_scale);
70 + }
71 +}
72 +
73 } // namespace vllm
74
75 void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
@@ -47,10 +79,10 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
79 TORCH_CHECK(out.is_contiguous());
80 TORCH_CHECK(scale.numel() == 1);
81
50 - int hidden_size = input.size(-1);
51 - int num_tokens = input.numel() / hidden_size;
52 - dim3 grid(num_tokens);
53 - dim3 block(std::min(hidden_size, 1024));
82 + int const hidden_size = input.size(-1);
83 + int const num_tokens = input.numel() / hidden_size;
84 + dim3 const grid(num_tokens);
85 + dim3 const block(std::min(hidden_size, 1024));
86 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
87 VLLM_DISPATCH_FLOATING_TYPES(
88 input.scalar_type(), "static_scaled_int8_quant_kernel", [&] {
@@ -60,3 +92,24 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
92 scale.data_ptr<float>(), hidden_size);
93 });
94 }
95 +
96 +void dynamic_scaled_int8_quant(
97 + torch::Tensor& out, // [..., hidden_size]
98 + torch::Tensor const& input, // [..., hidden_size]
99 + torch::Tensor& scales) {
100 + TORCH_CHECK(input.is_contiguous());
101 + TORCH_CHECK(out.is_contiguous());
102 +
103 + int const hidden_size = input.size(-1);
104 + int const num_tokens = input.numel() / hidden_size;
105 + dim3 const grid(num_tokens);
106 + dim3 const block(std::min(hidden_size, 1024));
107 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
108 + VLLM_DISPATCH_FLOATING_TYPES(
109 + input.scalar_type(), "dynamic_scaled_int8_quant_kernel", [&] {
110 + vllm::dynamic_scaled_int8_quant_kernel<scalar_t, float>
111 + <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
112 + out.data_ptr<int8_t>(),
113 + scales.data_ptr<float>(), hidden_size);
114 + });
115 +}
csrc/reduction_utils.cuh
+41 -13
@@ -21,29 +21,47 @@
21 #include "cuda_compat.h"
22
23 namespace vllm {
24 +
25 +namespace detail {
26 +
27 +template <typename T>
28 +__inline__ __device__ T _max(T a, T b) {
29 + return max(a, b);
30 +}
31 +
32 +template <typename T>
33 +__inline__ __device__ T _sum(T a, T b) {
34 + return a + b;
35 +}
36 +
37 +} // namespace detail
38 +
39 +template <typename T>
40 +using ReduceFnType = T (*)(T, T);
41 +
42 +// Helper function to return the next largest power of 2
43 +static constexpr int _nextPow2(unsigned int num) {
44 + if (num <= 1) return num;
45 + return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
46 +}
47 +
48 template <typename T, int numLanes = WARP_SIZE>
25 -__inline__ __device__ T warpReduceSum(T val) {
49 +__inline__ __device__ T warpReduce(T val, ReduceFnType<T> fn) {
50 static_assert(numLanes > 0 && (numLanes & (numLanes - 1)) == 0,
51 "numLanes is not a positive power of 2!");
52 static_assert(numLanes <= WARP_SIZE);
53 #pragma unroll
54 for (int mask = numLanes >> 1; mask > 0; mask >>= 1)
31 - val += VLLM_SHFL_XOR_SYNC(val, mask);
32 - return val;
33 -}
55 + val = fn(val, VLLM_SHFL_XOR_SYNC(val, mask));
56
35 -// Helper function to return the next largest power of 2
36 -static constexpr int _nextPow2(unsigned int num) {
37 - if (num <= 1) return num;
38 - return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
57 + return val;
58 }
59
41 -/* Calculate the sum of all elements in a block */
60 template <typename T, int maxBlockSize = 1024>
43 -__inline__ __device__ T blockReduceSum(T val) {
61 +__inline__ __device__ T blockReduce(T val, ReduceFnType<T> fn) {
62 static_assert(maxBlockSize <= 1024);
63 if constexpr (maxBlockSize > WARP_SIZE) {
46 - val = warpReduceSum<T>(val);
64 + val = warpReduce<T>(val, fn);
65 // Calculates max number of lanes that need to participate in the last
66 // warpReduce
67 constexpr int maxActiveLanes = (maxBlockSize + WARP_SIZE - 1) / WARP_SIZE;
@@ -56,12 +74,22 @@ __inline__ __device__ T blockReduceSum(T val) {
74
75 val = (threadIdx.x < blockDim.x / float(WARP_SIZE)) ? shared[lane]
76 : (T)(0.0f);
59 - val = warpReduceSum<T, _nextPow2(maxActiveLanes)>(val);
77 + val = warpReduce<T, _nextPow2(maxActiveLanes)>(val, fn);
78 } else {
79 // A single warpReduce is equal to blockReduce
62 - val = warpReduceSum<T, _nextPow2(maxBlockSize)>(val);
80 + val = warpReduce<T, _nextPow2(maxBlockSize)>(val, fn);
81 }
82 return val;
83 }
84
85 +template <typename T, int maxBlockSize = 1024>
86 +__inline__ __device__ T blockReduceMax(T val) {
87 + return blockReduce<T, maxBlockSize>(val, detail::_max<T>);
88 +}
89 +
90 +template <typename T, int maxBlockSize = 1024>
91 +__inline__ __device__ T blockReduceSum(T val) {
92 + return blockReduce<T, maxBlockSize>(val, detail::_sum<T>);
93 +}
94 +
95 } // namespace vllm
tests/kernels/test_int8_quant.py
+38 -6
@@ -4,27 +4,59 @@ import torch
4 from vllm._C import ops
5
6 DTYPES = [torch.half, torch.bfloat16, torch.float]
7 -HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 8192] # Arbitrary values for testing
7 +HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
8 + 8193] # Arbitrary values for testing
9 NUM_TOKENS = [1, 7, 83, 4096] # Arbitrary values for testing
10 SEEDS = [0]
11 SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
12
13
14 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
15 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
16 +@pytest.mark.parametrize("dtype", DTYPES)
17 +@pytest.mark.parametrize("seed", SEEDS)
18 +@torch.inference_mode()
19 +def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
20 + dtype: torch.dtype, seed: int) -> None:
21 + torch.random.manual_seed(seed)
22 + torch.cuda.manual_seed(seed)
23 + int8_traits = torch.iinfo(torch.int8)
24 +
25 + x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
26 +
27 + x_token_max, _ = x.max(dim=1)
28 + x_token_max = x_token_max.to(dtype=torch.float32)
29 + scales = (x_token_max / float(127.0))[:, None].to(device="cuda",
30 + dtype=torch.float32)
31 + torch_out = (x / scales).round().clamp(int8_traits.min,
32 + int8_traits.max).to(torch.int8)
33 +
34 + ops_out = torch.empty_like(x, dtype=torch.int8, device="cuda")
35 + scales_out = torch.empty_like(scales, dtype=torch.float32, device="cuda")
36 + ops.dynamic_scaled_int8_quant(ops_out, x, scales_out)
37 +
38 + assert torch.allclose(scales_out, scales)
39 + assert torch.allclose(torch_out, ops_out,
40 + atol=1) # big atol to account for rounding errors
41 +
42 +
43 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
44 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
45 @pytest.mark.parametrize("dtype", DTYPES)
46 @pytest.mark.parametrize("seed", SEEDS)
47 @pytest.mark.parametrize("scale", SCALE)
48 @torch.inference_mode()
19 -def test_quant(num_tokens: int, hidden_size: int, dtype: torch.dtype,
20 - seed: int, scale: float) -> None:
49 +def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
50 + dtype: torch.dtype, seed: int,
51 + scale: float) -> None:
52 torch.random.manual_seed(seed)
53 torch.cuda.manual_seed(seed)
54 + int8_traits = torch.iinfo(torch.int8)
55 +
56 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
57
25 - out1 = (x / scale).round().clamp(
26 - torch.iinfo(torch.int8).min,
27 - torch.iinfo(torch.int8).max).to(torch.int8)
58 + out1 = (x / scale).round().clamp(int8_traits.min,
59 + int8_traits.max).to(torch.int8)
60 out2 = torch.empty_like(x, dtype=torch.int8)
61 scale_argument = torch.tensor([scale], dtype=torch.float32, device="cuda")
62
tests/quantization/test_compressed_tensors.py
+18 -1
@@ -6,7 +6,8 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
6 import torch
7
8 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
9 - CompressedTensorsLinearMethod, CompressedTensorsW8A8StaticTensor)
9 + CompressedTensorsLinearMethod, CompressedTensorsW8A8DynamicToken,
10 + CompressedTensorsW8A8StaticTensor)
11
12
13 def test_compressed_tensors_w8a8_static_setup(vllm_runner):
@@ -34,3 +35,19 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner):
35 assert qkv_proj.weight_scale.shard_splitter is not None
36 assert qkv_proj.weight_scale.logical_widths is not None
37 assert qkv_proj.input_scale.dtype is torch.float32
38 +
39 +
40 +def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
41 + model_path = "nm-testing/tinyllama-one-shot-dynamic-test"
42 + llm = vllm_runner(model_path,
43 + quantization="sparseml",
44 + enforce_eager=True,
45 + dtype=torch.float16)
46 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
47 + layer = model.model.layers[0]
48 +
49 + qkv_proj = layer.self_attn.qkv_proj
50 +
51 + assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
52 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
53 + assert qkv_proj.weight.dtype is torch.int8
vllm/_custom_ops.py
+20 -8
@@ -266,21 +266,33 @@ def scaled_fp8_quant(
266
267
268 # int8
269 -def static_scaled_int8_quant(input: torch.Tensor,
270 - scale: torch.Tensor) -> torch.Tensor:
269 +def scaled_int8_quant(
270 + input: torch.Tensor,
271 + scale: Optional[torch.Tensor] = None
272 +) -> Tuple[torch.Tensor, torch.Tensor]:
273 """
272 - Quantize the input tensor to int8 and return the quantized tensor.
274 + Quantize the input tensor to int8 and return the quantized tensor and scale.
275
276 Args:
277 input: The input tensor to be quantized to int8.
276 - scale: Scaling factor for the int8 quantization.
278 + scale: Optional scaling factor for the int8 quantization.
279 + When not provided, we invoke dynamic-per-token quantization.
280
281 Returns:
279 - torch.Tensor: Output tensor in int8.
282 + Tuple[Torch.Tensor, Torch.Tensor] : Output int8 tensor and scales.
283 """
281 - q = torch.empty_like(input, dtype=torch.int8)
282 - vllm_ops.static_scaled_int8_quant(q, input, scale)
283 - return q
284 + output = torch.empty_like(input, dtype=torch.int8)
285 + if scale is not None:
286 + # static-per-tensor quantization.
287 + vllm_ops.static_scaled_int8_quant(output, input, scale)
288 + return output, scale
289 +
290 + # dynamic-per-token quantization.
291 + input_scales = torch.empty((input.numel() // input.shape[-1], 1),
292 + device=input.device,
293 + dtype=torch.float32)
294 + vllm_ops.dynamic_scaled_int8_quant(output, input, input_scales)
295 + return output, input_scales
296
297
298 # moe
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+51 -36
@@ -1,12 +1,16 @@
1 from typing import Any, Dict, List, Optional
2
3 import torch
4 +from pydantic import BaseModel
5
6 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7 from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8 QuantizationConfig)
9 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
9 - CompressedTensorsScheme, CompressedTensorsW8A8StaticTensor)
10 + CompressedTensorsScheme, CompressedTensorsW8A8DynamicToken,
11 + CompressedTensorsW8A8StaticTensor)
12 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
13 + QuantizationArgs, QuantizationStrategy, find_first_name_or_class_match)
14
15
16 class CompressedTensorsConfig(QuantizationConfig):
@@ -47,10 +51,12 @@ class CompressedTensorsConfig(QuantizationConfig):
51 targets = quant_config.get("targets")
52 for target in targets:
53 layer_quant_details[target] = {}
50 - layer_quant_details[target]["weight"] = quant_config.get(
51 - "weights")
52 - layer_quant_details[target]["input"] = quant_config.get(
53 - "input_activations")
54 + layer_quant_details[target][
55 + "weight"] = QuantizationArgs.parse_obj(
56 + quant_config.get("weights"))
57 + layer_quant_details[target][
58 + "input"] = QuantizationArgs.parse_obj(
59 + quant_config.get("input_activations"))
60
61 return cls(layer_quant_details=layer_quant_details, ignore=ignore)
62
@@ -58,40 +64,46 @@ class CompressedTensorsConfig(QuantizationConfig):
64 def get_config_filenames(cls) -> List[str]:
65 return []
66
61 - def _get_schema(self, weight_quant: Dict, input_quant: Dict):
62 - # TODO: Refactor as additional cases are supported
63 -
64 - weight_bit = weight_quant.get("num_bits")
65 - input_bit = input_quant.get("num_bits")
66 -
67 - weight_strategy = weight_quant.get("strategy")
68 - input_strategy = input_quant.get("strategy")
69 -
70 - weight_symmetric = weight_quant.get("symmetric")
71 - input_symmetric = input_quant.get("symmetric")
67 + def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
68 + input_quant: BaseModel) -> bool:
69 + is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
70 + is_tensor = (weight_quant.strategy == input_quant.strategy ==
71 + QuantizationStrategy.TENSOR.value)
72 + is_symmetric = weight_quant.symmetric and input_quant.symmetric
73 + is_static = not weight_quant.dynamic and not input_quant.dynamic
74 +
75 + return is_8_bits and is_tensor and is_symmetric and is_static
76 +
77 + def _is_dynamic_token_w8a8(self, weight_quant: BaseModel,
78 + input_quant: BaseModel) -> bool:
79 + is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
80 + is_token_tensor = (weight_quant.strategy
81 + == QuantizationStrategy.TENSOR.value) and (
82 + input_quant.strategy
83 + == QuantizationStrategy.TOKEN.value)
84 + is_symmetric = weight_quant.symmetric and input_quant.symmetric
85 + is_dynamic = not weight_quant.dynamic and input_quant.dynamic
86 +
87 + return is_8_bits and is_token_tensor and is_symmetric and is_dynamic
88 +
89 + def _get_schema(self, weight_quant: BaseModel,
90 + input_quant: BaseModel) -> "CompressedTensorsScheme":
91 + if self._is_static_tensor_w8a8(weight_quant, input_quant):
92 + return CompressedTensorsW8A8StaticTensor()
93
73 - is_8_bits = weight_bit == input_bit == 8
74 - is_tensor = weight_strategy == input_strategy == "tensor"
75 - is_symmetric = weight_symmetric and input_symmetric
94 + if self._is_dynamic_token_w8a8(weight_quant, input_quant):
95 + return CompressedTensorsW8A8DynamicToken()
96
77 - if is_8_bits and is_tensor and is_symmetric and \
78 - torch.cuda.is_available():
79 - # CompressedTensorsW8A8StaticTensor only supports CUDA path for
80 - # now.
81 - return CompressedTensorsW8A8StaticTensor()
82 - raise NotImplementedError(
83 - "Scheme not supported. Only CUDA, 8-bit static symmtetric "
84 - "per tensor quantization is currently supported")
97 + raise NotImplementedError("Scheme not supported.")
98
99 def get_scheme(self, layer: torch.nn.Module) -> "CompressedTensorsScheme":
100
88 - # TODO: update with matching function from `compressed_tensors`
89 - layer_type_name = None
90 - layer_name_class = type(layer).__name__.lower()
91 - for target in self.layer_quant_details:
92 - if target.lower() in layer_name_class:
93 - layer_type_name = target
94 - break
101 + layer_type_name = find_first_name_or_class_match(
102 + name="",
103 + module=layer,
104 + targets=self.layer_quant_details.keys(),
105 + check_contains=True)
106 +
107 if layer_type_name is None:
108 raise ValueError(f"Could not matching target for layer {layer}")
109
@@ -117,7 +129,9 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
129 **extra_weight_attrs):
130 """
131 Use the CompressedTensorsScheme associated with each layer to create
120 - the necessary parameters for the layer.
132 + the necessary parameters for the layer. See LinearMethodBase for param
133 + details
134 +
135 """
136 weight_loader = extra_weight_attrs.get("weight_loader")
137
@@ -139,7 +153,8 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
153 """
154 Use the output of create_weights and the CompressedTensorsScheme
155 associated with the layer to apply the forward pass with the
142 - layer input.
156 + layer input. See LinearMethodBase for param details
157 +
158 """
159
160 if bias is not None:
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+2
@@ -1,5 +1,7 @@
1 from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2 from .compressed_tensors_unquantized import ( # noqa: F401
3 CompressedTensorsUnquantized)
4 +from .compressed_tensors_w8a8_dynamictoken import ( # noqa: F401, E501
5 + CompressedTensorsW8A8DynamicToken)
6 from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
7 CompressedTensorsW8A8StaticTensor)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py new
+85
@@ -0,0 +1,85 @@
1 +from typing import Callable, List, Tuple, Union
2 +
3 +import torch
4 +from torch.nn import Parameter
5 +
6 +from vllm import _custom_ops as custom_ops
7 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 + CompressedTensorsScheme)
9 +from vllm.model_executor.utils import set_weight_attrs
10 +
11 +__all__ = ["CompressedTensorsW8A8DynamicToken"]
12 +
13 +
14 +class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
15 +
16 + def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
17 + if isinstance(shard_id, int):
18 + return shard_id
19 +
20 + assert isinstance(shard_id, str)
21 + qkv_idxs = {"q": 0, "k": 1, "v": 2}
22 + assert shard_id in qkv_idxs
23 + return qkv_idxs[shard_id]
24 +
25 + def scales_shard_splitter(
26 + self, param: torch.Tensor, loaded_weight: torch.Tensor,
27 + shard_id: Union[str, int],
28 + logical_widths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
29 + shard_id = self._shard_id_as_int(shard_id)
30 + offset = sum(logical_widths[:shard_id])
31 + size = logical_widths[shard_id]
32 + # update loaded weight with copies for broadcast.
33 + loaded_weight = loaded_weight.repeat(size)
34 + return param[offset:offset + size], loaded_weight
35 +
36 + def create_weights(self, layer: torch.nn.Module,
37 + output_partition_sizes: List[int],
38 + input_size_per_partition: int,
39 + params_dtype: torch.dtype, weight_loader: Callable,
40 + **kwargs):
41 +
42 + # When the scales have a single value, it is required that they be
43 + # on the CPU for performance and CUDA Graphs compatibility. Please
44 + # refer to the comment in
45 + # CompressedTensorsW8A8StaticTensor::create_weights for further
46 + # information.
47 + is_tensor_partitioned = len(output_partition_sizes) != 1
48 + weight_scale_dim = sum(
49 + output_partition_sizes) if is_tensor_partitioned else 1
50 +
51 + weight_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
52 + requires_grad=False)
53 +
54 + weight_scale = Parameter(torch.empty(weight_scale_dim,
55 + dtype=torch.float32),
56 + requires_grad=False)
57 +
58 + weight = Parameter(torch.empty(sum(output_partition_sizes),
59 + input_size_per_partition,
60 + dtype=torch.int8),
61 + requires_grad=False)
62 +
63 + layer.register_parameter("weight", weight)
64 + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
65 + set_weight_attrs(weight, {"weight_loader": weight_loader})
66 + set_weight_attrs(weight, {"logical_widths": output_partition_sizes})
67 +
68 + layer.register_parameter("weight_scale", weight_scale)
69 + set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
70 + set_weight_attrs(
71 + weight_scale, {
72 + "shard_splitter": self.scales_shard_splitter,
73 + "logical_widths": output_partition_sizes
74 + })
75 +
76 + layer.register_parameter("weight_zero_point", weight_zero_point)
77 + set_weight_attrs(weight_zero_point, {"weight_loader": weight_loader})
78 +
79 + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
80 + weight = layer.weight
81 + weight_scale = layer.weight_scale
82 +
83 + x_q, input_scales = custom_ops.scaled_int8_quant(x)
84 + return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), input_scales,
85 + weight_scale, x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+1 -1
@@ -97,7 +97,7 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
97 act_scale = layer.input_scale
98
99 # Input quantize
100 - x_q = custom_ops.static_scaled_int8_quant(x, act_scale)
100 + x_q, _ = custom_ops.scaled_int8_quant(x, act_scale)
101
102 return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), act_scale,
103 weight_scale, x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/utils.py new
+114
@@ -0,0 +1,114 @@
1 +import re
2 +from enum import Enum
3 +from typing import Any, Dict, Iterable, Optional
4 +
5 +from pydantic import BaseModel, Field
6 +from torch.nn import Module
7 +
8 +
9 +class QuantizationType(str, Enum):
10 + """
11 + Enum storing quantization type options
12 + """
13 +
14 + INT = "int"
15 + FLOAT = "float"
16 +
17 +
18 +class QuantizationStrategy(str, Enum):
19 + """
20 + Enum storing quantization strategy options
21 + """
22 +
23 + TENSOR = "tensor"
24 + CHANNEL = "channel"
25 + GROUP = "group"
26 + BLOCK = "block"
27 + TOKEN = "token"
28 +
29 +
30 +class QuantizationArgs(BaseModel):
31 + """
32 + User facing arguments used to define a quantization config
33 + for weights or activations
34 +
35 + :param num_bits: quantization bit depth
36 + :param type: dtype to quantized to, either int or float
37 + :param symmetric: whether or not quantization scale is symmetric
38 + :param strategy: string determining the scope of scale/zero-point to apply
39 + :param group_size: group length to use for the group strategy
40 + :param block_structure: 2d block structure to use for the block
41 + strategy, must be of the format "2x4", "8x16", etc.
42 + :param dynamic: set True to perform dynamic quantization -
43 + values will not be calibrated during calibration phase,
44 + instead during inference new quantization ranges will be
45 + observed with every sample. Defaults to False for static
46 + quantization. Note that enabling dynamic quantization
47 + will change the default observer to a memoryless one
48 + """
49 +
50 + num_bits: int = 8
51 + type: QuantizationType = QuantizationType.INT
52 + symmetric: bool = True
53 + group_size: Optional[int] = None
54 + strategy: Optional[QuantizationStrategy] = None
55 + block_structure: Optional[str] = None
56 + dynamic: bool = False
57 + observer: str = Field(
58 + default="minmax",
59 + description=("The class to use to compute the quantization param - "
60 + "scale and zero-point'"),
61 + )
62 + observer_kwargs: Dict[str, Any] = Field(
63 + default_factory=dict,
64 + description=
65 + ("optional dict of kwargs to be passed directly to torch quantization "
66 + "Observers constructor excluding quantization range or symmetry"),
67 + )
68 +
69 +
70 +def find_first_name_or_class_match(
71 + name: str,
72 + module: Module,
73 + targets: Iterable[str],
74 + check_contains: bool = False) -> Optional[str]:
75 + """
76 + Helper function to map the quantization details listed in the config
77 + for a given list of targets against each model layer. First uses the
78 + layer name to try and find a match. If no name match is found, uses
79 + the layer class name. Returns None otherwise.
80 +
81 + :param name: layer name
82 + :param module: torch.nn.Module
83 + :param targets: list of targets to match the layer against
84 + :param check_contains: whether or not to do a substring match
85 + """
86 +
87 + return _find_first_match(name, targets) or _find_first_match(
88 + module.__class__.__name__, targets, check_contains)
89 +
90 +
91 +def _find_first_match(value: str,
92 + targets: Iterable[str],
93 + check_contains: bool = False) -> Optional[str]:
94 + """
95 + Returns first element of target that matches value either
96 + exactly or as a regex after 're:'. If check_contains is set to True,
97 + additionally checks if the target string is contained within the value.
98 +
99 + :param value: string to compare the list of targets against
100 + :param targets: list of targets to match the layer against
101 + :param check_contains: whether or not to do a substring match
102 + """
103 +
104 + for target in targets:
105 + if target.startswith("re:"):
106 + pattern = target[3:]
107 + if re.match(pattern, value):
108 + return target
109 + elif check_contains:
110 + if target.lower() in value.lower():
111 + return target
112 + elif target == value:
113 + return target
114 + return None