@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fea59c7712
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix][Kernel] Use int64_t for indices in fp8 quant kernels (#6649)

Tyler Michael Smith committed Jul 22, 2024 at 16:08 UTC fea59c7712eea40f09d36fa732f0b9a7024ea983
2 files changed +31 -6
csrc/quantization/fp8/common.cu
+6 -6
@@ -103,11 +103,11 @@ __device__ float thread_max_vec(scalar_t const* __restrict__ input,
103 vec4_t<scalar_t> const* vectorized_in =
104 reinterpret_cast<vec4_t<scalar_t> const*>(input);
105
106 - int const num_vec_elems = num_elems >> 2;
106 + int64_t const num_vec_elems = num_elems >> 2;
107 float absmax_val = 0.0f;
108
109 #pragma unroll 4
110 - for (int i = tid; i < num_vec_elems; i += step) {
110 + for (int64_t i = tid; i < num_vec_elems; i += step) {
111 vec4_t<scalar_t> in_vec = vectorized_in[i];
112 absmax_val = max(absmax_val, fabs(in_vec.x));
113 absmax_val = max(absmax_val, fabs(in_vec.y));
@@ -116,7 +116,7 @@ __device__ float thread_max_vec(scalar_t const* __restrict__ input,
116 }
117
118 // Handle the remaining elements if num_elems is not divisible by 4
119 - for (int i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
119 + for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
120 absmax_val = max(absmax_val, fabs(input[i]));
121 }
122
@@ -134,10 +134,10 @@ __device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
134 reinterpret_cast<vec4_t<scalar_t> const*>(input);
135 float8x4_t* vectorized_out = reinterpret_cast<float8x4_t*>(out);
136
137 - int const num_vec_elems = num_elems >> 2;
137 + int64_t const num_vec_elems = num_elems >> 2;
138
139 #pragma unroll 4
140 - for (int i = tid; i < num_vec_elems; i += step) {
140 + for (int64_t i = tid; i < num_vec_elems; i += step) {
141 vec4_t<scalar_t> in_vec = vectorized_in[i];
142 float8x4_t out_vec;
143
@@ -153,7 +153,7 @@ __device__ void scaled_fp8_conversion_vec(c10::Float8_e4m3fn* __restrict__ out,
153 }
154
155 // Handle the remaining elements if num_elems is not divisible by 4
156 - for (int i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
156 + for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
157 out[i] = scaled_fp8_conversion<is_scale_inverted>(
158 static_cast<float>(input[i]), scale);
159 }
tests/kernels/test_fp8_quant.py
+25
@@ -60,3 +60,28 @@ def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
60 assert torch.allclose(ref_scale, ops_scale)
61 assert torch.allclose(ref_out.to(dtype=torch.float32),
62 ops_out.to(dtype=torch.float32))
63 +
64 +
65 +# Regression test for a case with large activations where an int32 index cannot
66 +# represent the number of elements.
67 +@torch.inference_mode()
68 +@pytest.mark.parametrize("seed", SEEDS)
69 +def test_fp8_quant_large(seed: int) -> None:
70 + torch.random.manual_seed(seed)
71 + torch.cuda.manual_seed(seed)
72 +
73 + num_tokens = 1024000 # Mistral-Nemo's max_position_embeddings
74 + hidden_size = 1152 # Smallest hidden_size to reproduce the error
75 + dtype = torch.bfloat16
76 +
77 + x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
78 + ref_out, scale = ref_dynamic_per_tensor_fp8_quant(x)
79 + ops_out, _ = ops.scaled_fp8_quant(x, scale)
80 +
81 + # Minimize memory footprint in this test by freeing x and upconverting
82 + # the outputs in place. (torch.allclose does not support fp8)
83 + del x
84 + ref_out = ref_out.to(dtype=dtype)
85 + ops_out = ops_out.to(dtype=dtype)
86 +
87 + assert torch.allclose(ref_out, ops_out)