@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 92d86da217
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[BugFix] [Kernel] Fix GPU SEGV occurring in int8 kernels (#9391)

rasmith committed Oct 16, 2024 at 20:34 UTC 92d86da217c38f7e033fc56936a9db32a97c03bd
1 file changed +28 -14
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+28 -14
@@ -96,12 +96,15 @@ __global__ void static_scaled_int8_quant_kernel(
96 scalar_t const* __restrict__ input, int8_t* __restrict__ out,
97 scale_type const* scale_ptr, const int hidden_size) {
98 int const tid = threadIdx.x;
99 - int const token_idx = blockIdx.x;
99 + int64_t const token_idx = blockIdx.x;
100 scale_type const scale = *scale_ptr;
101
102 + // Must be performed using 64-bit math to avoid integer overflow.
103 + out += token_idx * hidden_size;
104 + input += token_idx * hidden_size;
105 +
106 for (int i = tid; i < hidden_size; i += blockDim.x) {
103 - out[token_idx * hidden_size + i] = float_to_int8_rn(
104 - static_cast<float>(input[token_idx * hidden_size + i]) / scale);
107 + out[i] = float_to_int8_rn(static_cast<float>(input[i]) / scale);
108 }
109 }
110
@@ -111,14 +114,18 @@ __global__ void static_scaled_int8_azp_quant_kernel(
114 scale_type const* scale_ptr, azp_type const* azp_ptr,
115 const int hidden_size) {
116 int const tid = threadIdx.x;
114 - int const token_idx = blockIdx.x;
117 + int64_t const token_idx = blockIdx.x;
118 scale_type const scale = *scale_ptr;
119 azp_type const azp = *azp_ptr;
120
121 + // Must be performed using 64-bit math to avoid integer overflow.
122 + out += token_idx * hidden_size;
123 + input += token_idx * hidden_size;
124 +
125 for (int i = tid; i < hidden_size; i += blockDim.x) {
119 - auto const val = static_cast<float>(input[token_idx * hidden_size + i]);
126 + auto const val = static_cast<float>(input[i]);
127 auto const quant_val = int32_to_int8(float_to_int32_rn(val / scale) + azp);
121 - out[token_idx * hidden_size + i] = quant_val;
128 + out[i] = quant_val;
129 }
130 }
131
@@ -127,12 +134,16 @@ __global__ void dynamic_scaled_int8_quant_kernel(
134 scalar_t const* __restrict__ input, int8_t* __restrict__ out,
135 scale_type* scale, const int hidden_size) {
136 int const tid = threadIdx.x;
130 - int const token_idx = blockIdx.x;
137 + int64_t const token_idx = blockIdx.x;
138 float absmax_val = 0.0f;
139 float const zero = 0.0f;
140
141 + // Must be performed using 64-bit math to avoid integer overflow.
142 + out += token_idx * hidden_size;
143 + input += token_idx * hidden_size;
144 +
145 for (int i = tid; i < hidden_size; i += blockDim.x) {
135 - float val = static_cast<float>(input[token_idx * hidden_size + i]);
146 + float val = static_cast<float>(input[i]);
147 val = val > zero ? val : -val;
148 absmax_val = val > absmax_val ? val : absmax_val;
149 }
@@ -150,8 +161,7 @@ __global__ void dynamic_scaled_int8_quant_kernel(
161
162 float const tmp_scale = 127.0f / block_absmax_val;
163 for (int i = tid; i < hidden_size; i += blockDim.x) {
153 - out[token_idx * hidden_size + i] = float_to_int8_rn(
154 - static_cast<float>(input[token_idx * hidden_size + i]) * tmp_scale);
164 + out[i] = float_to_int8_rn(static_cast<float>(input[i]) * tmp_scale);
165 }
166 }
167
@@ -159,13 +169,17 @@ template <typename scalar_t, typename scale_type, typename azp_type>
169 __global__ void dynamic_scaled_int8_azp_quant_kernel(
170 scalar_t const* __restrict__ input, int8_t* __restrict__ out,
171 scale_type* scale, azp_type* azp, const int hidden_size) {
162 - int const token_idx = blockIdx.x;
172 + int64_t const token_idx = blockIdx.x;
173 +
174 + // Must be performed using 64-bit math to avoid integer overflow.
175 + out += token_idx * hidden_size;
176 + input += token_idx * hidden_size;
177
178 // Scan for the min and max value for this token
179 float max_val = std::numeric_limits<float>::min();
180 float min_val = std::numeric_limits<float>::max();
181 for (int i = threadIdx.x; i < hidden_size; i += blockDim.x) {
168 - auto val = static_cast<float>(input[token_idx * hidden_size + i]);
182 + auto val = static_cast<float>(input[i]);
183 max_val = std::max(max_val, val);
184 min_val = std::min(min_val, val);
185 }
@@ -200,10 +214,10 @@ __global__ void dynamic_scaled_int8_azp_quant_kernel(
214
215 // Quantize the values
216 for (int i = threadIdx.x; i < hidden_size; i += blockDim.x) {
203 - auto const val = static_cast<float>(input[token_idx * hidden_size + i]);
217 + auto const val = static_cast<float>(input[i]);
218 auto const quant_val =
219 int32_to_int8(float_to_int32_rn(val / scale_val) + azp_val);
206 - out[token_idx * hidden_size + i] = quant_val;
220 + out[i] = quant_val;
221 }
222 }
223