A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Dynamic Per-Token Activation Quantization (#5037)
Co-authored-by: Varun Sundar Rabindranath <varunsundar08@gmail.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Dipika Sikka committed
Jun 7, 2024 at 12:36 UTC
ca3ea51bde6c22d0afb3aa0a3fdba6d568095a0a
12 files changed
+440
-76
csrc/ops.h
+3
@@ -97,6 +97,9 @@ int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
97
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
98
torch::Tensor const& scale);
99
100
+void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
101
+ torch::Tensor& scales);
102
+
103
void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
104
torch::Tensor lookup_table);
105
csrc/pybind.cpp
+3
@@ -70,6 +70,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
70
ops.def("static_scaled_int8_quant", &static_scaled_int8_quant,
71
"Compute int8 quantized tensor for given scaling factor");
72
73
+ ops.def("dynamic_scaled_int8_quant", &dynamic_scaled_int8_quant,
74
+ "Compute int8 quantized tensor and scaling factor");
75
+
76
// Cache ops
77
pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
78
cache_ops.def("swap_blocks", &swap_blocks,
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+64
-11
@@ -3,6 +3,7 @@
3
#include <cmath>
4
5
#include "../../dispatch_utils.h"
6
+#include "../../reduction_utils.cuh"
7
8
static inline __device__ int8_t float_to_int8_rn(float x) {
9
#ifdef USE_ROCM
@@ -27,17 +28,48 @@ namespace vllm {
28
29
template <typename scalar_t, typename scale_type>
30
__global__ void static_scaled_int8_quant_kernel(
30
- const scalar_t* __restrict__ input, int8_t* __restrict__ out,
31
- const scale_type* scale_ptr, const int hidden_size) {
32
- const int tid = threadIdx.x;
33
- const int token_idx = blockIdx.x;
34
- scale_type scale = *scale_ptr;
31
+ scalar_t const* __restrict__ input, int8_t* __restrict__ out,
32
+ scale_type const* scale_ptr, const int hidden_size) {
33
+ int const tid = threadIdx.x;
34
+ int const token_idx = blockIdx.x;
35
+ scale_type const scale = *scale_ptr;
36
37
for (int i = tid; i < hidden_size; i += blockDim.x) {
37
- out[token_idx * hidden_size + i] =
38
- float_to_int8_rn(((float)input[token_idx * hidden_size + i]) / scale);
38
+ out[token_idx * hidden_size + i] = float_to_int8_rn(
39
+ static_cast<float>(input[token_idx * hidden_size + i]) / scale);
40
}
41
}
42
+
43
+template <typename scalar_t, typename scale_type>
44
+__global__ void dynamic_scaled_int8_quant_kernel(
45
+ scalar_t const* __restrict__ input, int8_t* __restrict__ out,
46
+ scale_type* scale, const int hidden_size) {
47
+ int const tid = threadIdx.x;
48
+ int const token_idx = blockIdx.x;
49
+ float absmax_val = 0.0f;
50
+ float const zero = 0.0f;
51
+
52
+ for (int i = tid; i < hidden_size; i += blockDim.x) {
53
+ float val = static_cast<float>(input[token_idx * hidden_size + i]);
54
+ val = val > zero ? val : -val;
55
+ absmax_val = val > absmax_val ? val : absmax_val;
56
+ }
57
+
58
+ float const block_absmax_val_maybe = blockReduceMax(absmax_val);
59
+ __shared__ float block_absmax_val;
60
+ if (tid == 0) {
61
+ block_absmax_val = block_absmax_val_maybe;
62
+ scale[token_idx] = block_absmax_val / 127.0f;
63
+ }
64
+ __syncthreads();
65
+
66
+ float const tmp_scale = 127.0f / block_absmax_val;
67
+ for (int i = tid; i < hidden_size; i += blockDim.x) {
68
+ out[token_idx * hidden_size + i] = float_to_int8_rn(
69
+ static_cast<float>(input[token_idx * hidden_size + i]) * tmp_scale);
70
+ }
71
+}
72
+
73
} // namespace vllm
74
75
void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
@@ -47,10 +79,10 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
79
TORCH_CHECK(out.is_contiguous());
80
TORCH_CHECK(scale.numel() == 1);
81
50
- int hidden_size = input.size(-1);
51
- int num_tokens = input.numel() / hidden_size;
52
- dim3 grid(num_tokens);
53
- dim3 block(std::min(hidden_size, 1024));
82
+ int const hidden_size = input.size(-1);
83
+ int const num_tokens = input.numel() / hidden_size;
84
+ dim3 const grid(num_tokens);
85
+ dim3 const block(std::min(hidden_size, 1024));
86
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
87
VLLM_DISPATCH_FLOATING_TYPES(
88
input.scalar_type(), "static_scaled_int8_quant_kernel", [&] {
@@ -60,3 +92,24 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
92
scale.data_ptr<float>(), hidden_size);
93
});
94
}
95
+
96
+void dynamic_scaled_int8_quant(
97
+ torch::Tensor& out, // [..., hidden_size]
98
+ torch::Tensor const& input, // [..., hidden_size]
99
+ torch::Tensor& scales) {
100
+ TORCH_CHECK(input.is_contiguous());
101
+ TORCH_CHECK(out.is_contiguous());
102
+
103
+ int const hidden_size = input.size(-1);
104
+ int const num_tokens = input.numel() / hidden_size;
105
+ dim3 const grid(num_tokens);
106
+ dim3 const block(std::min(hidden_size, 1024));
107
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
108
+ VLLM_DISPATCH_FLOATING_TYPES(
109
+ input.scalar_type(), "dynamic_scaled_int8_quant_kernel", [&] {
110
+ vllm::dynamic_scaled_int8_quant_kernel<scalar_t, float>
111
+ <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
112
+ out.data_ptr<int8_t>(),
113
+ scales.data_ptr<float>(), hidden_size);
114
+ });
115
+}
csrc/reduction_utils.cuh
+41
-13
@@ -21,29 +21,47 @@
21
#include "cuda_compat.h"
22
23
namespace vllm {
24
+
25
+namespace detail {
26
+
27
+template <typename T>
28
+__inline__ __device__ T _max(T a, T b) {
29
+ return max(a, b);
30
+}
31
+
32
+template <typename T>
33
+__inline__ __device__ T _sum(T a, T b) {
34
+ return a + b;
35
+}
36
+
37
+} // namespace detail
38
+
39
+template <typename T>
40
+using ReduceFnType = T (*)(T, T);
41
+
42
+// Helper function to return the next largest power of 2
43
+static constexpr int _nextPow2(unsigned int num) {
44
+ if (num <= 1) return num;
45
+ return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
46
+}
47
+
48
template <typename T, int numLanes = WARP_SIZE>
25
-__inline__ __device__ T warpReduceSum(T val) {
49
+__inline__ __device__ T warpReduce(T val, ReduceFnType<T> fn) {
50
static_assert(numLanes > 0 && (numLanes & (numLanes - 1)) == 0,
51
"numLanes is not a positive power of 2!");
52
static_assert(numLanes <= WARP_SIZE);
53
#pragma unroll
54
for (int mask = numLanes >> 1; mask > 0; mask >>= 1)
31
- val += VLLM_SHFL_XOR_SYNC(val, mask);
32
- return val;
33
-}
55
+ val = fn(val, VLLM_SHFL_XOR_SYNC(val, mask));
56
35
-// Helper function to return the next largest power of 2
36
-static constexpr int _nextPow2(unsigned int num) {
37
- if (num <= 1) return num;
38
- return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
57
+ return val;
58
}
59
41
-/* Calculate the sum of all elements in a block */
60
template <typename T, int maxBlockSize = 1024>
43
-__inline__ __device__ T blockReduceSum(T val) {
61
+__inline__ __device__ T blockReduce(T val, ReduceFnType<T> fn) {
62
static_assert(maxBlockSize <= 1024);
63
if constexpr (maxBlockSize > WARP_SIZE) {
46
- val = warpReduceSum<T>(val);
64
+ val = warpReduce<T>(val, fn);
65
// Calculates max number of lanes that need to participate in the last
66
// warpReduce
67
constexpr int maxActiveLanes = (maxBlockSize + WARP_SIZE - 1) / WARP_SIZE;
@@ -56,12 +74,22 @@ __inline__ __device__ T blockReduceSum(T val) {
74
75
val = (threadIdx.x < blockDim.x / float(WARP_SIZE)) ? shared[lane]
76
: (T)(0.0f);
59
- val = warpReduceSum<T, _nextPow2(maxActiveLanes)>(val);
77
+ val = warpReduce<T, _nextPow2(maxActiveLanes)>(val, fn);
78
} else {
79
// A single warpReduce is equal to blockReduce
62
- val = warpReduceSum<T, _nextPow2(maxBlockSize)>(val);
80
+ val = warpReduce<T, _nextPow2(maxBlockSize)>(val, fn);
81
}
82
return val;
83
}
84
85
+template <typename T, int maxBlockSize = 1024>
86
+__inline__ __device__ T blockReduceMax(T val) {
87
+ return blockReduce<T, maxBlockSize>(val, detail::_max<T>);
88
+}
89
+
90
+template <typename T, int maxBlockSize = 1024>
91
+__inline__ __device__ T blockReduceSum(T val) {
92
+ return blockReduce<T, maxBlockSize>(val, detail::_sum<T>);
93
+}
94
+
95
} // namespace vllm
tests/kernels/test_int8_quant.py
+38
-6
@@ -4,27 +4,59 @@ import torch
4
from vllm._C import ops
5
6
DTYPES = [torch.half, torch.bfloat16, torch.float]
7
-HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 8192] # Arbitrary values for testing
7
+HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
8
+ 8193] # Arbitrary values for testing
9
NUM_TOKENS = [1, 7, 83, 4096] # Arbitrary values for testing
10
SEEDS = [0]
11
SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
12
13
14
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
15
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
16
+@pytest.mark.parametrize("dtype", DTYPES)
17
+@pytest.mark.parametrize("seed", SEEDS)
18
+@torch.inference_mode()
19
+def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
20
+ dtype: torch.dtype, seed: int) -> None:
21
+ torch.random.manual_seed(seed)
22
+ torch.cuda.manual_seed(seed)
23
+ int8_traits = torch.iinfo(torch.int8)
24
+
25
+ x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
26
+
27
+ x_token_max, _ = x.max(dim=1)
28
+ x_token_max = x_token_max.to(dtype=torch.float32)
29
+ scales = (x_token_max / float(127.0))[:, None].to(device="cuda",
30
+ dtype=torch.float32)
31
+ torch_out = (x / scales).round().clamp(int8_traits.min,
32
+ int8_traits.max).to(torch.int8)
33
+
34
+ ops_out = torch.empty_like(x, dtype=torch.int8, device="cuda")
35
+ scales_out = torch.empty_like(scales, dtype=torch.float32, device="cuda")
36
+ ops.dynamic_scaled_int8_quant(ops_out, x, scales_out)
37
+
38
+ assert torch.allclose(scales_out, scales)
39
+ assert torch.allclose(torch_out, ops_out,
40
+ atol=1) # big atol to account for rounding errors
41
+
42
+
43
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
44
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
45
@pytest.mark.parametrize("dtype", DTYPES)
46
@pytest.mark.parametrize("seed", SEEDS)
47
@pytest.mark.parametrize("scale", SCALE)
48
@torch.inference_mode()
19
-def test_quant(num_tokens: int, hidden_size: int, dtype: torch.dtype,
20
- seed: int, scale: float) -> None:
49
+def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
50
+ dtype: torch.dtype, seed: int,
51
+ scale: float) -> None:
52
torch.random.manual_seed(seed)
53
torch.cuda.manual_seed(seed)
54
+ int8_traits = torch.iinfo(torch.int8)
55
+
56
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
57
25
- out1 = (x / scale).round().clamp(
26
- torch.iinfo(torch.int8).min,
27
- torch.iinfo(torch.int8).max).to(torch.int8)
58
+ out1 = (x / scale).round().clamp(int8_traits.min,
59
+ int8_traits.max).to(torch.int8)
60
out2 = torch.empty_like(x, dtype=torch.int8)
61
scale_argument = torch.tensor([scale], dtype=torch.float32, device="cuda")
62
tests/quantization/test_compressed_tensors.py
+18
-1
@@ -6,7 +6,8 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
6
import torch
7
8
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
9
- CompressedTensorsLinearMethod, CompressedTensorsW8A8StaticTensor)
9
+ CompressedTensorsLinearMethod, CompressedTensorsW8A8DynamicToken,
10
+ CompressedTensorsW8A8StaticTensor)
11
12
13
def test_compressed_tensors_w8a8_static_setup(vllm_runner):
@@ -34,3 +35,19 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner):
35
assert qkv_proj.weight_scale.shard_splitter is not None
36
assert qkv_proj.weight_scale.logical_widths is not None
37
assert qkv_proj.input_scale.dtype is torch.float32
38
+
39
+
40
+def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
41
+ model_path = "nm-testing/tinyllama-one-shot-dynamic-test"
42
+ llm = vllm_runner(model_path,
43
+ quantization="sparseml",
44
+ enforce_eager=True,
45
+ dtype=torch.float16)
46
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model
47
+ layer = model.model.layers[0]
48
+
49
+ qkv_proj = layer.self_attn.qkv_proj
50
+
51
+ assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
52
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
53
+ assert qkv_proj.weight.dtype is torch.int8
vllm/_custom_ops.py
+20
-8
@@ -266,21 +266,33 @@ def scaled_fp8_quant(
266
267
268
# int8
269
-def static_scaled_int8_quant(input: torch.Tensor,
270
- scale: torch.Tensor) -> torch.Tensor:
269
+def scaled_int8_quant(
270
+ input: torch.Tensor,
271
+ scale: Optional[torch.Tensor] = None
272
+) -> Tuple[torch.Tensor, torch.Tensor]:
273
"""
272
- Quantize the input tensor to int8 and return the quantized tensor.
274
+ Quantize the input tensor to int8 and return the quantized tensor and scale.
275
276
Args:
277
input: The input tensor to be quantized to int8.
276
- scale: Scaling factor for the int8 quantization.
278
+ scale: Optional scaling factor for the int8 quantization.
279
+ When not provided, we invoke dynamic-per-token quantization.
280
281
Returns:
279
- torch.Tensor: Output tensor in int8.
282
+ Tuple[Torch.Tensor, Torch.Tensor] : Output int8 tensor and scales.
283
"""
281
- q = torch.empty_like(input, dtype=torch.int8)
282
- vllm_ops.static_scaled_int8_quant(q, input, scale)
283
- return q
284
+ output = torch.empty_like(input, dtype=torch.int8)
285
+ if scale is not None:
286
+ # static-per-tensor quantization.
287
+ vllm_ops.static_scaled_int8_quant(output, input, scale)
288
+ return output, scale
289
+
290
+ # dynamic-per-token quantization.
291
+ input_scales = torch.empty((input.numel() // input.shape[-1], 1),
292
+ device=input.device,
293
+ dtype=torch.float32)
294
+ vllm_ops.dynamic_scaled_int8_quant(output, input, input_scales)
295
+ return output, input_scales
296
297
298
# moe
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+51
-36
@@ -1,12 +1,16 @@
1
from typing import Any, Dict, List, Optional
2
3
import torch
4
+from pydantic import BaseModel
5
6
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7
from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8
QuantizationConfig)
9
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
9
- CompressedTensorsScheme, CompressedTensorsW8A8StaticTensor)
10
+ CompressedTensorsScheme, CompressedTensorsW8A8DynamicToken,
11
+ CompressedTensorsW8A8StaticTensor)
12
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
13
+ QuantizationArgs, QuantizationStrategy, find_first_name_or_class_match)
14
15
16
class CompressedTensorsConfig(QuantizationConfig):
@@ -47,10 +51,12 @@ class CompressedTensorsConfig(QuantizationConfig):
51
targets = quant_config.get("targets")
52
for target in targets:
53
layer_quant_details[target] = {}
50
- layer_quant_details[target]["weight"] = quant_config.get(
51
- "weights")
52
- layer_quant_details[target]["input"] = quant_config.get(
53
- "input_activations")
54
+ layer_quant_details[target][
55
+ "weight"] = QuantizationArgs.parse_obj(
56
+ quant_config.get("weights"))
57
+ layer_quant_details[target][
58
+ "input"] = QuantizationArgs.parse_obj(
59
+ quant_config.get("input_activations"))
60
61
return cls(layer_quant_details=layer_quant_details, ignore=ignore)
62
@@ -58,40 +64,46 @@ class CompressedTensorsConfig(QuantizationConfig):
64
def get_config_filenames(cls) -> List[str]:
65
return []
66
61
- def _get_schema(self, weight_quant: Dict, input_quant: Dict):
62
- # TODO: Refactor as additional cases are supported
63
-
64
- weight_bit = weight_quant.get("num_bits")
65
- input_bit = input_quant.get("num_bits")
66
-
67
- weight_strategy = weight_quant.get("strategy")
68
- input_strategy = input_quant.get("strategy")
69
-
70
- weight_symmetric = weight_quant.get("symmetric")
71
- input_symmetric = input_quant.get("symmetric")
67
+ def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
68
+ input_quant: BaseModel) -> bool:
69
+ is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
70
+ is_tensor = (weight_quant.strategy == input_quant.strategy ==
71
+ QuantizationStrategy.TENSOR.value)
72
+ is_symmetric = weight_quant.symmetric and input_quant.symmetric
73
+ is_static = not weight_quant.dynamic and not input_quant.dynamic
74
+
75
+ return is_8_bits and is_tensor and is_symmetric and is_static
76
+
77
+ def _is_dynamic_token_w8a8(self, weight_quant: BaseModel,
78
+ input_quant: BaseModel) -> bool:
79
+ is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
80
+ is_token_tensor = (weight_quant.strategy
81
+ == QuantizationStrategy.TENSOR.value) and (
82
+ input_quant.strategy
83
+ == QuantizationStrategy.TOKEN.value)
84
+ is_symmetric = weight_quant.symmetric and input_quant.symmetric
85
+ is_dynamic = not weight_quant.dynamic and input_quant.dynamic
86
+
87
+ return is_8_bits and is_token_tensor and is_symmetric and is_dynamic
88
+
89
+ def _get_schema(self, weight_quant: BaseModel,
90
+ input_quant: BaseModel) -> "CompressedTensorsScheme":
91
+ if self._is_static_tensor_w8a8(weight_quant, input_quant):
92
+ return CompressedTensorsW8A8StaticTensor()
93
73
- is_8_bits = weight_bit == input_bit == 8
74
- is_tensor = weight_strategy == input_strategy == "tensor"
75
- is_symmetric = weight_symmetric and input_symmetric
94
+ if self._is_dynamic_token_w8a8(weight_quant, input_quant):
95
+ return CompressedTensorsW8A8DynamicToken()
96
77
- if is_8_bits and is_tensor and is_symmetric and \
78
- torch.cuda.is_available():
79
- # CompressedTensorsW8A8StaticTensor only supports CUDA path for
80
- # now.
81
- return CompressedTensorsW8A8StaticTensor()
82
- raise NotImplementedError(
83
- "Scheme not supported. Only CUDA, 8-bit static symmtetric "
84
- "per tensor quantization is currently supported")
97
+ raise NotImplementedError("Scheme not supported.")
98
99
def get_scheme(self, layer: torch.nn.Module) -> "CompressedTensorsScheme":
100
88
- # TODO: update with matching function from `compressed_tensors`
89
- layer_type_name = None
90
- layer_name_class = type(layer).__name__.lower()
91
- for target in self.layer_quant_details:
92
- if target.lower() in layer_name_class:
93
- layer_type_name = target
94
- break
101
+ layer_type_name = find_first_name_or_class_match(
102
+ name="",
103
+ module=layer,
104
+ targets=self.layer_quant_details.keys(),
105
+ check_contains=True)
106
+
107
if layer_type_name is None:
108
raise ValueError(f"Could not matching target for layer {layer}")
109
@@ -117,7 +129,9 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
129
**extra_weight_attrs):
130
"""
131
Use the CompressedTensorsScheme associated with each layer to create
120
- the necessary parameters for the layer.
132
+ the necessary parameters for the layer. See LinearMethodBase for param
133
+ details
134
+
135
"""
136
weight_loader = extra_weight_attrs.get("weight_loader")
137
@@ -139,7 +153,8 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
153
"""
154
Use the output of create_weights and the CompressedTensorsScheme
155
associated with the layer to apply the forward pass with the
142
- layer input.
156
+ layer input. See LinearMethodBase for param details
157
+
158
"""
159
160
if bias is not None:
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+2
@@ -1,5 +1,7 @@
1
from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2
from .compressed_tensors_unquantized import ( # noqa: F401
3
CompressedTensorsUnquantized)
4
+from .compressed_tensors_w8a8_dynamictoken import ( # noqa: F401, E501
5
+ CompressedTensorsW8A8DynamicToken)
6
from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
7
CompressedTensorsW8A8StaticTensor)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py
new
+85
@@ -0,0 +1,85 @@
1
+from typing import Callable, List, Tuple, Union
2
+
3
+import torch
4
+from torch.nn import Parameter
5
+
6
+from vllm import _custom_ops as custom_ops
7
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
+ CompressedTensorsScheme)
9
+from vllm.model_executor.utils import set_weight_attrs
10
+
11
+__all__ = ["CompressedTensorsW8A8DynamicToken"]
12
+
13
+
14
+class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
15
+
16
+ def _shard_id_as_int(self, shard_id: Union[str, int]) -> int:
17
+ if isinstance(shard_id, int):
18
+ return shard_id
19
+
20
+ assert isinstance(shard_id, str)
21
+ qkv_idxs = {"q": 0, "k": 1, "v": 2}
22
+ assert shard_id in qkv_idxs
23
+ return qkv_idxs[shard_id]
24
+
25
+ def scales_shard_splitter(
26
+ self, param: torch.Tensor, loaded_weight: torch.Tensor,
27
+ shard_id: Union[str, int],
28
+ logical_widths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
29
+ shard_id = self._shard_id_as_int(shard_id)
30
+ offset = sum(logical_widths[:shard_id])
31
+ size = logical_widths[shard_id]
32
+ # update loaded weight with copies for broadcast.
33
+ loaded_weight = loaded_weight.repeat(size)
34
+ return param[offset:offset + size], loaded_weight
35
+
36
+ def create_weights(self, layer: torch.nn.Module,
37
+ output_partition_sizes: List[int],
38
+ input_size_per_partition: int,
39
+ params_dtype: torch.dtype, weight_loader: Callable,
40
+ **kwargs):
41
+
42
+ # When the scales have a single value, it is required that they be
43
+ # on the CPU for performance and CUDA Graphs compatibility. Please
44
+ # refer to the comment in
45
+ # CompressedTensorsW8A8StaticTensor::create_weights for further
46
+ # information.
47
+ is_tensor_partitioned = len(output_partition_sizes) != 1
48
+ weight_scale_dim = sum(
49
+ output_partition_sizes) if is_tensor_partitioned else 1
50
+
51
+ weight_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
52
+ requires_grad=False)
53
+
54
+ weight_scale = Parameter(torch.empty(weight_scale_dim,
55
+ dtype=torch.float32),
56
+ requires_grad=False)
57
+
58
+ weight = Parameter(torch.empty(sum(output_partition_sizes),
59
+ input_size_per_partition,
60
+ dtype=torch.int8),
61
+ requires_grad=False)
62
+
63
+ layer.register_parameter("weight", weight)
64
+ set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
65
+ set_weight_attrs(weight, {"weight_loader": weight_loader})
66
+ set_weight_attrs(weight, {"logical_widths": output_partition_sizes})
67
+
68
+ layer.register_parameter("weight_scale", weight_scale)
69
+ set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
70
+ set_weight_attrs(
71
+ weight_scale, {
72
+ "shard_splitter": self.scales_shard_splitter,
73
+ "logical_widths": output_partition_sizes
74
+ })
75
+
76
+ layer.register_parameter("weight_zero_point", weight_zero_point)
77
+ set_weight_attrs(weight_zero_point, {"weight_loader": weight_loader})
78
+
79
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
80
+ weight = layer.weight
81
+ weight_scale = layer.weight_scale
82
+
83
+ x_q, input_scales = custom_ops.scaled_int8_quant(x)
84
+ return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), input_scales,
85
+ weight_scale, x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+1
-1
@@ -97,7 +97,7 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
97
act_scale = layer.input_scale
98
99
# Input quantize
100
- x_q = custom_ops.static_scaled_int8_quant(x, act_scale)
100
+ x_q, _ = custom_ops.scaled_int8_quant(x, act_scale)
101
102
return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), act_scale,
103
weight_scale, x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
new
+114
@@ -0,0 +1,114 @@
1
+import re
2
+from enum import Enum
3
+from typing import Any, Dict, Iterable, Optional
4
+
5
+from pydantic import BaseModel, Field
6
+from torch.nn import Module
7
+
8
+
9
+class QuantizationType(str, Enum):
10
+ """
11
+ Enum storing quantization type options
12
+ """
13
+
14
+ INT = "int"
15
+ FLOAT = "float"
16
+
17
+
18
+class QuantizationStrategy(str, Enum):
19
+ """
20
+ Enum storing quantization strategy options
21
+ """
22
+
23
+ TENSOR = "tensor"
24
+ CHANNEL = "channel"
25
+ GROUP = "group"
26
+ BLOCK = "block"
27
+ TOKEN = "token"
28
+
29
+
30
+class QuantizationArgs(BaseModel):
31
+ """
32
+ User facing arguments used to define a quantization config
33
+ for weights or activations
34
+
35
+ :param num_bits: quantization bit depth
36
+ :param type: dtype to quantized to, either int or float
37
+ :param symmetric: whether or not quantization scale is symmetric
38
+ :param strategy: string determining the scope of scale/zero-point to apply
39
+ :param group_size: group length to use for the group strategy
40
+ :param block_structure: 2d block structure to use for the block
41
+ strategy, must be of the format "2x4", "8x16", etc.
42
+ :param dynamic: set True to perform dynamic quantization -
43
+ values will not be calibrated during calibration phase,
44
+ instead during inference new quantization ranges will be
45
+ observed with every sample. Defaults to False for static
46
+ quantization. Note that enabling dynamic quantization
47
+ will change the default observer to a memoryless one
48
+ """
49
+
50
+ num_bits: int = 8
51
+ type: QuantizationType = QuantizationType.INT
52
+ symmetric: bool = True
53
+ group_size: Optional[int] = None
54
+ strategy: Optional[QuantizationStrategy] = None
55
+ block_structure: Optional[str] = None
56
+ dynamic: bool = False
57
+ observer: str = Field(
58
+ default="minmax",
59
+ description=("The class to use to compute the quantization param - "
60
+ "scale and zero-point'"),
61
+ )
62
+ observer_kwargs: Dict[str, Any] = Field(
63
+ default_factory=dict,
64
+ description=
65
+ ("optional dict of kwargs to be passed directly to torch quantization "
66
+ "Observers constructor excluding quantization range or symmetry"),
67
+ )
68
+
69
+
70
+def find_first_name_or_class_match(
71
+ name: str,
72
+ module: Module,
73
+ targets: Iterable[str],
74
+ check_contains: bool = False) -> Optional[str]:
75
+ """
76
+ Helper function to map the quantization details listed in the config
77
+ for a given list of targets against each model layer. First uses the
78
+ layer name to try and find a match. If no name match is found, uses
79
+ the layer class name. Returns None otherwise.
80
+
81
+ :param name: layer name
82
+ :param module: torch.nn.Module
83
+ :param targets: list of targets to match the layer against
84
+ :param check_contains: whether or not to do a substring match
85
+ """
86
+
87
+ return _find_first_match(name, targets) or _find_first_match(
88
+ module.__class__.__name__, targets, check_contains)
89
+
90
+
91
+def _find_first_match(value: str,
92
+ targets: Iterable[str],
93
+ check_contains: bool = False) -> Optional[str]:
94
+ """
95
+ Returns first element of target that matches value either
96
+ exactly or as a regex after 're:'. If check_contains is set to True,
97
+ additionally checks if the target string is contained within the value.
98
+
99
+ :param value: string to compare the list of targets against
100
+ :param targets: list of targets to match the layer against
101
+ :param check_contains: whether or not to do a substring match
102
+ """
103
+
104
+ for target in targets:
105
+ if target.startswith("re:"):
106
+ pattern = target[3:]
107
+ if re.match(pattern, value):
108
+ return target
109
+ elif check_contains:
110
+ if target.lower() in value.lower():
111
+ return target
112
+ elif target == value:
113
+ return target
114
+ return None