A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Kernel] Replaced `blockReduce[...]` functions with `cub::BlockReduce` (#7233)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Luka Govedič committed
Aug 21, 2024 at 20:18 UTC
7937009a7e82c3c4c9c7f48d11142bee5aac4a30
8 files changed
+237
-116
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml
+2
-2
@@ -4,8 +4,8 @@ tasks:
4
- name: "gsm8k"
5
metrics:
6
- name: "exact_match,strict-match"
7
- value: 0.409
7
+ value: 0.419
8
- name: "exact_match,flexible-extract"
9
- value: 0.406
9
+ value: 0.416
10
limit: 1000
11
num_fewshot: 5
benchmarks/kernels/benchmark_layernorm.py
new
+89
@@ -0,0 +1,89 @@
1
+import random
2
+import time
3
+
4
+import torch
5
+
6
+from vllm.model_executor.layers.layernorm import RMSNorm
7
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser
8
+
9
+
10
+@torch.inference_mode()
11
+def main(num_tokens: int,
12
+ hidden_size: int,
13
+ add_residual: bool,
14
+ dtype: torch.dtype,
15
+ seed: int = 0,
16
+ do_profile: bool = False,
17
+ num_warmup_iters: int = 5,
18
+ num_iters: int = 100) -> None:
19
+ random.seed(seed)
20
+ torch.random.manual_seed(seed)
21
+ if torch.cuda.is_available():
22
+ torch.cuda.manual_seed(seed)
23
+ torch.set_default_device("cuda")
24
+
25
+ layer = RMSNorm(hidden_size).to(dtype=dtype)
26
+ layer.weight.data.normal_(mean=1.0, std=0.1)
27
+ scale = 1 / (2 * hidden_size)
28
+ x = torch.randn(num_tokens, hidden_size, dtype=dtype)
29
+ x *= scale
30
+ residual = torch.randn_like(x) * scale if add_residual else None
31
+
32
+ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float:
33
+ torch.cuda.synchronize()
34
+ if profile:
35
+ torch.cuda.cudart().cudaProfilerStart()
36
+ start_time = time.perf_counter()
37
+
38
+ for _ in range(num_iters):
39
+ layer(x, residual)
40
+ torch.cuda.synchronize()
41
+
42
+ end_time = time.perf_counter()
43
+ if profile:
44
+ torch.cuda.cudart().cudaProfilerStart()
45
+ return (end_time - start_time) / num_iters
46
+
47
+ # Warmup.
48
+ print("Warming up...")
49
+ run_benchmark = run_cuda_benchmark
50
+ run_benchmark(num_iters=num_warmup_iters, profile=False)
51
+
52
+ # Benchmark.
53
+ if do_profile:
54
+ latency = run_benchmark(num_iters=1, profile=True)
55
+ else:
56
+ latency = run_benchmark(num_iters=num_iters, profile=False)
57
+ print(f"Kernel running time: {latency * 1000000:.3f} us")
58
+
59
+
60
+if __name__ == '__main__':
61
+ parser = FlexibleArgumentParser(
62
+ description="Benchmark the layernorm kernel.")
63
+ parser.add_argument("--num-tokens", type=int, default=4096)
64
+ parser.add_argument("--hidden-size", type=int, default=8192)
65
+ parser.add_argument("--add-residual", action="store_true")
66
+ parser.add_argument("--dtype",
67
+ type=str,
68
+ choices=["half", "bfloat16", "float"],
69
+ default="half")
70
+ parser.add_argument("--seed", type=int, default=0)
71
+ parser.add_argument("--profile", action="store_true")
72
+ parser.add_argument("--num-warmup-iters", type=int, default=5)
73
+ parser.add_argument("--num-iters",
74
+ type=int,
75
+ default=100,
76
+ help="Number of benchmark iterations. "
77
+ "If --profile is set, this number is ignored")
78
+
79
+ args = parser.parse_args()
80
+ print(args)
81
+
82
+ main(num_tokens=args.num_tokens,
83
+ hidden_size=args.hidden_size,
84
+ add_residual=args.add_residual,
85
+ dtype=STR_DTYPE_TO_TORCH_DTYPE[args.dtype],
86
+ seed=args.seed,
87
+ do_profile=args.profile,
88
+ num_warmup_iters=args.num_warmup_iters,
89
+ num_iters=args.num_iters)
benchmarks/kernels/benchmark_quant.py
new
+103
@@ -0,0 +1,103 @@
1
+import random
2
+import time
3
+
4
+import torch
5
+
6
+from vllm import _custom_ops as ops
7
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser
8
+
9
+
10
+@torch.inference_mode()
11
+def main(num_tokens: int,
12
+ hidden_size: int,
13
+ static_scale: bool,
14
+ quant_dtype: torch.dtype,
15
+ dtype: torch.dtype,
16
+ seed: int = 0,
17
+ do_profile: bool = False,
18
+ num_warmup_iters: int = 5,
19
+ num_iters: int = 100) -> None:
20
+ random.seed(seed)
21
+ torch.random.manual_seed(seed)
22
+ if torch.cuda.is_available():
23
+ torch.cuda.manual_seed(seed)
24
+ torch.set_default_device("cuda")
25
+
26
+ x = torch.randn(num_tokens, hidden_size, dtype=dtype)
27
+ scale = torch.randn(1, 1, dtype=torch.float32) if static_scale else None
28
+
29
+ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float:
30
+ torch.cuda.synchronize()
31
+ if profile:
32
+ torch.cuda.cudart().cudaProfilerStart()
33
+ start_time = time.perf_counter()
34
+
35
+ for _ in range(num_iters):
36
+ if quant_dtype == torch.int8:
37
+ ops.scaled_int8_quant(x, scale)
38
+ else:
39
+ ops.scaled_fp8_quant(x, scale)
40
+ torch.cuda.synchronize()
41
+
42
+ end_time = time.perf_counter()
43
+ if profile:
44
+ torch.cuda.cudart().cudaProfilerStart()
45
+ return (end_time - start_time) / num_iters
46
+
47
+ # Warmup.
48
+ print("Warming up...")
49
+ run_benchmark = run_cuda_benchmark
50
+ run_benchmark(num_iters=num_warmup_iters, profile=False)
51
+
52
+ # Benchmark.
53
+ if do_profile:
54
+ latency = run_benchmark(num_iters=1, profile=True)
55
+ else:
56
+ latency = run_benchmark(num_iters=num_iters, profile=False)
57
+ print(f"Kernel running time: {latency * 1000000:.3f} us")
58
+
59
+
60
+if __name__ == '__main__':
61
+
62
+ def to_torch_dtype(dt):
63
+ if dt == "int8":
64
+ return torch.int8
65
+ if dt == "fp8":
66
+ return torch.float8_e4m3fn
67
+ raise ValueError(f"Unsupported dtype: {dt}")
68
+
69
+ parser = FlexibleArgumentParser(
70
+ description="Benchmark the quantization (fp8 or int8) kernel.")
71
+ parser.add_argument("--num-tokens", type=int, default=4096)
72
+ parser.add_argument("--hidden-size", type=int, default=8192)
73
+ parser.add_argument("--static-scale", action="store_true")
74
+ parser.add_argument("--quant-dtype",
75
+ type=str,
76
+ choices=["fp8", "int8"],
77
+ default="int8")
78
+ parser.add_argument("--dtype",
79
+ type=str,
80
+ choices=["half", "bfloat16", "float"],
81
+ default="half")
82
+
83
+ parser.add_argument("--seed", type=int, default=0)
84
+ parser.add_argument("--profile", action="store_true")
85
+ parser.add_argument("--num-warmup-iters", type=int, default=5)
86
+ parser.add_argument("--num-iters",
87
+ type=int,
88
+ default=100,
89
+ help="Number of benchmark iterations. "
90
+ "If --profile is set, this number is ignored")
91
+
92
+ args = parser.parse_args()
93
+ print(args)
94
+
95
+ main(num_tokens=args.num_tokens,
96
+ hidden_size=args.hidden_size,
97
+ static_scale=args.static_scale,
98
+ quant_dtype=to_torch_dtype(args.quant_dtype),
99
+ dtype=STR_DTYPE_TO_TORCH_DTYPE[args.dtype],
100
+ seed=args.seed,
101
+ do_profile=args.profile,
102
+ num_warmup_iters=args.num_warmup_iters,
103
+ num_iters=args.num_iters)
csrc/layernorm_kernels.cu
+19
-14
@@ -3,13 +3,16 @@
3
#include <c10/cuda/CUDAGuard.h>
4
5
#include "dispatch_utils.h"
6
-#include "reduction_utils.cuh"
6
#ifndef USE_ROCM
7
#include <cuda_bf16.h>
8
#include <cuda_fp16.h>
9
+ #include <cub/util_type.cuh>
10
+ #include <cub/cub.cuh>
11
#else
12
#include <hip/hip_bf16.h>
13
#include <hip/hip_fp16.h>
14
+ #include <hipcub/util_type.hpp>
15
+ #include <hipcub/hipcub.hpp>
16
17
using __nv_bfloat16 = __hip_bfloat16;
18
using __nv_bfloat162 = __hip_bfloat162;
@@ -31,7 +34,11 @@ __global__ void rms_norm_kernel(
34
const float x = (float)input[blockIdx.x * hidden_size + idx];
35
variance += x * x;
36
}
34
- variance = blockReduceSum<float>(variance);
37
+
38
+ using BlockReduce = cub::BlockReduce<float, 1024>;
39
+ __shared__ typename BlockReduce::TempStorage reduceStore;
40
+ variance = BlockReduce(reduceStore).Reduce(variance, cub::Sum{}, blockDim.x);
41
+
42
if (threadIdx.x == 0) {
43
s_variance = rsqrtf(variance / hidden_size + epsilon);
44
}
@@ -228,12 +235,11 @@ fused_add_rms_norm_kernel(
235
variance += temp.sum_squares();
236
residual_v[id] = temp;
237
}
231
- /* Keep the following if-else block in sync with the
232
- calculation of max_block_size in fused_add_rms_norm */
233
- if (num_tokens < 256) {
234
- variance = blockReduceSum<float, 1024>(variance);
235
- } else
236
- variance = blockReduceSum<float, 256>(variance);
238
+
239
+ using BlockReduce = cub::BlockReduce<float, 1024>;
240
+ __shared__ typename BlockReduce::TempStorage reduceStore;
241
+ variance = BlockReduce(reduceStore).Reduce(variance, cub::Sum{}, blockDim.x);
242
+
243
if (threadIdx.x == 0) {
244
s_variance = rsqrtf(variance / hidden_size + epsilon);
245
}
@@ -268,12 +274,11 @@ fused_add_rms_norm_kernel(
274
variance += x * x;
275
residual[blockIdx.x * hidden_size + idx] = z;
276
}
271
- /* Keep the following if-else block in sync with the
272
- calculation of max_block_size in fused_add_rms_norm */
273
- if (num_tokens < 256) {
274
- variance = blockReduceSum<float, 1024>(variance);
275
- } else
276
- variance = blockReduceSum<float, 256>(variance);
277
+
278
+ using BlockReduce = cub::BlockReduce<float, 1024>;
279
+ __shared__ typename BlockReduce::TempStorage reduceStore;
280
+ variance = BlockReduce(reduceStore).Reduce(variance, cub::Sum{}, blockDim.x);
281
+
282
if (threadIdx.x == 0) {
283
s_variance = rsqrtf(variance / hidden_size + epsilon);
284
}
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+12
-2
@@ -3,7 +3,14 @@
3
#include <cmath>
4
5
#include "../../dispatch_utils.h"
6
-#include "../../reduction_utils.cuh"
6
+
7
+#ifndef USE_ROCM
8
+ #include <cub/util_type.cuh>
9
+ #include <cub/cub.cuh>
10
+#else
11
+ #include <hipcub/util_type.hpp>
12
+ #include <hipcub/hipcub.hpp>
13
+#endif
14
15
static inline __device__ int8_t float_to_int8_rn(float x) {
16
#ifdef USE_ROCM
@@ -55,7 +62,10 @@ __global__ void dynamic_scaled_int8_quant_kernel(
62
absmax_val = val > absmax_val ? val : absmax_val;
63
}
64
58
- float const block_absmax_val_maybe = blockReduceMax(absmax_val);
65
+ using BlockReduce = cub::BlockReduce<float, 1024>;
66
+ __shared__ typename BlockReduce::TempStorage reduceStorage;
67
+ float const block_absmax_val_maybe =
68
+ BlockReduce(reduceStorage).Reduce(absmax_val, cub::Max{}, blockDim.x);
69
__shared__ float block_absmax_val;
70
if (tid == 0) {
71
block_absmax_val = block_absmax_val_maybe;
csrc/quantization/fp8/common.cu
+11
-2
@@ -7,7 +7,13 @@
7
#include "cuda_compat.h"
8
#include "dispatch_utils.h"
9
10
-#include "../../reduction_utils.cuh"
10
+#ifndef USE_ROCM
11
+ #include <cub/util_type.cuh>
12
+ #include <cub/cub.cuh>
13
+#else
14
+ #include <hipcub/util_type.hpp>
15
+ #include <hipcub/hipcub.hpp>
16
+#endif
17
18
#ifndef USE_ROCM
19
using FP8_TYPE = c10::Float8_e4m3fn;
@@ -215,7 +221,10 @@ __global__ void dynamic_per_token_scaled_fp8_quant_kernel(
221
}
222
}
223
218
- float const block_absmax_val_maybe = blockReduceMax(absmax_val);
224
+ using BlockReduce = cub::BlockReduce<float, 1024>;
225
+ __shared__ typename BlockReduce::TempStorage reduceStorage;
226
+ float const block_absmax_val_maybe =
227
+ BlockReduce(reduceStorage).Reduce(absmax_val, cub::Max{}, blockDim.x);
228
__shared__ float token_scale;
229
if (tid == 0) {
230
if (scale_ub) {
csrc/reduction_utils.cuh
deleted
-95
@@ -1,95 +0,0 @@
1
-/*
2
- * Adapted from
3
- * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/reduce_kernel_utils.cuh
4
- * Copyright (c) 2023, The vLLM team.
5
- * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
- *
7
- * Licensed under the Apache License, Version 2.0 (the "License");
8
- * you may not use this file except in compliance with the License.
9
- * You may obtain a copy of the License at
10
- *
11
- * http://www.apache.org/licenses/LICENSE-2.0
12
- *
13
- * Unless required by applicable law or agreed to in writing, software
14
- * distributed under the License is distributed on an "AS IS" BASIS,
15
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
- * See the License for the specific language governing permissions and
17
- * limitations under the License.
18
- */
19
-#pragma once
20
-
21
-#include "cuda_compat.h"
22
-
23
-namespace vllm {
24
-
25
-namespace detail {
26
-
27
-template <typename T>
28
-__inline__ __device__ T _max(T a, T b) {
29
- return max(a, b);
30
-}
31
-
32
-template <typename T>
33
-__inline__ __device__ T _sum(T a, T b) {
34
- return a + b;
35
-}
36
-
37
-} // namespace detail
38
-
39
-template <typename T>
40
-using ReduceFnType = T (*)(T, T);
41
-
42
-// Helper function to return the next largest power of 2
43
-static constexpr int _nextPow2(unsigned int num) {
44
- if (num <= 1) return num;
45
- return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
46
-}
47
-
48
-template <typename T, int numLanes = WARP_SIZE>
49
-__inline__ __device__ T warpReduce(T val, ReduceFnType<T> fn) {
50
- static_assert(numLanes > 0 && (numLanes & (numLanes - 1)) == 0,
51
- "numLanes is not a positive power of 2!");
52
- static_assert(numLanes <= WARP_SIZE);
53
-#pragma unroll
54
- for (int mask = numLanes >> 1; mask > 0; mask >>= 1)
55
- val = fn(val, VLLM_SHFL_XOR_SYNC(val, mask));
56
-
57
- return val;
58
-}
59
-
60
-template <typename T, int maxBlockSize = 1024>
61
-__inline__ __device__ T blockReduce(T val, ReduceFnType<T> fn) {
62
- static_assert(maxBlockSize <= 1024);
63
- if constexpr (maxBlockSize > WARP_SIZE) {
64
- val = warpReduce<T>(val, fn);
65
- // Calculates max number of lanes that need to participate in the last
66
- // warpReduce
67
- constexpr int maxActiveLanes = (maxBlockSize + WARP_SIZE - 1) / WARP_SIZE;
68
- static __shared__ T shared[maxActiveLanes];
69
- int lane = threadIdx.x % WARP_SIZE;
70
- int wid = threadIdx.x / WARP_SIZE;
71
- if (lane == 0) shared[wid] = val;
72
-
73
- __syncthreads();
74
-
75
- val = (threadIdx.x < blockDim.x / float(WARP_SIZE)) ? shared[lane]
76
- : (T)(0.0f);
77
- val = warpReduce<T, _nextPow2(maxActiveLanes)>(val, fn);
78
- } else {
79
- // A single warpReduce is equal to blockReduce
80
- val = warpReduce<T, _nextPow2(maxBlockSize)>(val, fn);
81
- }
82
- return val;
83
-}
84
-
85
-template <typename T, int maxBlockSize = 1024>
86
-__inline__ __device__ T blockReduceMax(T val) {
87
- return blockReduce<T, maxBlockSize>(val, detail::_max<T>);
88
-}
89
-
90
-template <typename T, int maxBlockSize = 1024>
91
-__inline__ __device__ T blockReduceSum(T val) {
92
- return blockReduce<T, maxBlockSize>(val, detail::_sum<T>);
93
-}
94
-
95
-} // namespace vllm
tests/basic_correctness/test_chunked_prefill.py
+1
-1
@@ -83,7 +83,7 @@ def test_models(
83
for m in E4M3_KV_MODELS])
84
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
85
@pytest.mark.parametrize("max_tokens", [4])
86
-@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16])
86
+@pytest.mark.parametrize("chunked_prefill_token_size", [4, 16])
87
@pytest.mark.parametrize("enforce_eager", [False, True])
88
# NOTE: Increasing this in this suite will fail CI because we currently cannot
89
# reset distributed env properly. Use a value > 1 just when you test.