A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Pass a device pointer into the quantize kernel for the scales (#5159)
Tyler Michael Smith committed
Jun 3, 2024 at 12:52 UTC
cbb2f59cc853731f5607ac0130bb6cdebfdc89c7
5 files changed
+16
-11
csrc/ops.h
+2
-2
@@ -94,8 +94,8 @@ int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
94
95
#endif
96
97
-void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor& input,
98
- float scale);
97
+void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
98
+ torch::Tensor const& scale);
99
100
void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
101
torch::Tensor lookup_table);
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+9
-6
@@ -28,9 +28,10 @@ namespace vllm {
28
template <typename scalar_t, typename scale_type>
29
__global__ void static_scaled_int8_quant_kernel(
30
const scalar_t* __restrict__ input, int8_t* __restrict__ out,
31
- scale_type scale, const int hidden_size) {
31
+ const scale_type* scale_ptr, const int hidden_size) {
32
const int tid = threadIdx.x;
33
const int token_idx = blockIdx.x;
34
+ scale_type scale = *scale_ptr;
35
36
for (int i = tid; i < hidden_size; i += blockDim.x) {
37
out[token_idx * hidden_size + i] =
@@ -39,11 +40,13 @@ __global__ void static_scaled_int8_quant_kernel(
40
}
41
} // namespace vllm
42
42
-void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
43
- torch::Tensor& input, // [..., hidden_size]
44
- float scale) {
43
+void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
44
+ torch::Tensor const& input, // [..., hidden_size]
45
+ torch::Tensor const& scale) {
46
TORCH_CHECK(input.is_contiguous());
47
TORCH_CHECK(out.is_contiguous());
48
+ TORCH_CHECK(scale.numel() == 1);
49
+
50
int hidden_size = input.size(-1);
51
int num_tokens = input.numel() / hidden_size;
52
dim3 grid(num_tokens);
@@ -53,7 +56,7 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
56
input.scalar_type(), "static_scaled_int8_quant_kernel", [&] {
57
vllm::static_scaled_int8_quant_kernel<scalar_t, float>
58
<<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
56
- out.data_ptr<int8_t>(), scale,
57
- hidden_size);
59
+ out.data_ptr<int8_t>(),
60
+ scale.data_ptr<float>(), hidden_size);
61
});
62
}
tests/kernels/test_int8_quant.py
+3
-1
@@ -26,6 +26,8 @@ def test_quant(num_tokens: int, hidden_size: int, dtype: torch.dtype,
26
torch.iinfo(torch.int8).min,
27
torch.iinfo(torch.int8).max).to(torch.int8)
28
out2 = torch.empty_like(x, dtype=torch.int8)
29
- ops.static_scaled_int8_quant(out2, x, scale)
29
+ scale_argument = torch.tensor([scale], dtype=torch.float32, device="cuda")
30
+
31
+ ops.static_scaled_int8_quant(out2, x, scale_argument)
32
assert torch.allclose(out1, out2,
33
atol=1) # big atol to account for rounding errors
vllm/_custom_ops.py
+1
-1
@@ -265,7 +265,7 @@ def scaled_fp8_quant(
265
266
# int8
267
def static_scaled_int8_quant(input: torch.Tensor,
268
- scale: float) -> torch.Tensor:
268
+ scale: torch.Tensor) -> torch.Tensor:
269
"""
270
Quantize the input tensor to int8 and return the quantized tensor.
271
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+1
-1
@@ -97,7 +97,7 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
97
act_scale = layer.input_scale
98
99
# Input quantize
100
- x_q = custom_ops.static_scaled_int8_quant(x, act_scale[0].item())
100
+ x_q = custom_ops.static_scaled_int8_quant(x, act_scale)
101
102
return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), act_scale,
103
weight_scale, x.dtype)