@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cbb2f59cc8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Pass a device pointer into the quantize kernel for the scales (#5159)

Tyler Michael Smith committed Jun 3, 2024 at 12:52 UTC cbb2f59cc853731f5607ac0130bb6cdebfdc89c7
5 files changed +16 -11
csrc/ops.h
+2 -2
@@ -94,8 +94,8 @@ int cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
94
95 #endif
96
97 -void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor& input,
98 - float scale);
97 +void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
98 + torch::Tensor const& scale);
99
100 void squeezellm_gemm(torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,
101 torch::Tensor lookup_table);
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+9 -6
@@ -28,9 +28,10 @@ namespace vllm {
28 template <typename scalar_t, typename scale_type>
29 __global__ void static_scaled_int8_quant_kernel(
30 const scalar_t* __restrict__ input, int8_t* __restrict__ out,
31 - scale_type scale, const int hidden_size) {
31 + const scale_type* scale_ptr, const int hidden_size) {
32 const int tid = threadIdx.x;
33 const int token_idx = blockIdx.x;
34 + scale_type scale = *scale_ptr;
35
36 for (int i = tid; i < hidden_size; i += blockDim.x) {
37 out[token_idx * hidden_size + i] =
@@ -39,11 +40,13 @@ __global__ void static_scaled_int8_quant_kernel(
40 }
41 } // namespace vllm
42
42 -void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
43 - torch::Tensor& input, // [..., hidden_size]
44 - float scale) {
43 +void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
44 + torch::Tensor const& input, // [..., hidden_size]
45 + torch::Tensor const& scale) {
46 TORCH_CHECK(input.is_contiguous());
47 TORCH_CHECK(out.is_contiguous());
48 + TORCH_CHECK(scale.numel() == 1);
49 +
50 int hidden_size = input.size(-1);
51 int num_tokens = input.numel() / hidden_size;
52 dim3 grid(num_tokens);
@@ -53,7 +56,7 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
56 input.scalar_type(), "static_scaled_int8_quant_kernel", [&] {
57 vllm::static_scaled_int8_quant_kernel<scalar_t, float>
58 <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
56 - out.data_ptr<int8_t>(), scale,
57 - hidden_size);
59 + out.data_ptr<int8_t>(),
60 + scale.data_ptr<float>(), hidden_size);
61 });
62 }
tests/kernels/test_int8_quant.py
+3 -1
@@ -26,6 +26,8 @@ def test_quant(num_tokens: int, hidden_size: int, dtype: torch.dtype,
26 torch.iinfo(torch.int8).min,
27 torch.iinfo(torch.int8).max).to(torch.int8)
28 out2 = torch.empty_like(x, dtype=torch.int8)
29 - ops.static_scaled_int8_quant(out2, x, scale)
29 + scale_argument = torch.tensor([scale], dtype=torch.float32, device="cuda")
30 +
31 + ops.static_scaled_int8_quant(out2, x, scale_argument)
32 assert torch.allclose(out1, out2,
33 atol=1) # big atol to account for rounding errors
vllm/_custom_ops.py
+1 -1
@@ -265,7 +265,7 @@ def scaled_fp8_quant(
265
266 # int8
267 def static_scaled_int8_quant(input: torch.Tensor,
268 - scale: float) -> torch.Tensor:
268 + scale: torch.Tensor) -> torch.Tensor:
269 """
270 Quantize the input tensor to int8 and return the quantized tensor.
271
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+1 -1
@@ -97,7 +97,7 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
97 act_scale = layer.input_scale
98
99 # Input quantize
100 - x_q = custom_ops.static_scaled_int8_quant(x, act_scale[0].item())
100 + x_q = custom_ops.static_scaled_int8_quant(x, act_scale)
101
102 return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), act_scale,
103 weight_scale, x.dtype)