A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Avoid compiling kernels for double data type (#933)
Woosuk Kwon committed
Sep 2, 2023 at 14:59 UTC
8ce9c50d4034de3c557b520935fac1d6dac585a0
5 files changed
+28
-21
csrc/activation_kernels.cu
+4
-6
@@ -1,6 +1,8 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
+#include "dispatch_utils.h"
5
+
6
namespace vllm {
7
8
template<typename T>
@@ -34,9 +36,7 @@ void silu_and_mul(
36
dim3 grid(num_tokens);
37
dim3 block(std::min(d, 1024));
38
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
37
- AT_DISPATCH_FLOATING_TYPES_AND2(
38
- at::ScalarType::Half,
39
- at::ScalarType::BFloat16,
39
+ VLLM_DISPATCH_FLOATING_TYPES(
40
input.scalar_type(),
41
"silu_and_mul_kernel",
42
[&] {
@@ -71,9 +71,7 @@ __global__ void activation_kernel(
71
dim3 grid(num_tokens); \
72
dim3 block(std::min(d, 1024)); \
73
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
74
- AT_DISPATCH_FLOATING_TYPES_AND2( \
75
- at::ScalarType::Half, \
76
- at::ScalarType::BFloat16, \
74
+ VLLM_DISPATCH_FLOATING_TYPES( \
75
input.scalar_type(), \
76
"activation_kernel", \
77
[&] { \
csrc/cache_kernels.cu
+5
-9
@@ -1,6 +1,8 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
+#include "dispatch_utils.h"
5
+
6
#include <algorithm>
7
#include <cassert>
8
#include <map>
@@ -125,9 +127,7 @@ void copy_blocks(
127
dim3 grid(num_layers, num_pairs);
128
dim3 block(std::min(1024, numel_per_block));
129
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
128
- AT_DISPATCH_FLOATING_TYPES_AND2(
129
- at::ScalarType::Half,
130
- at::ScalarType::BFloat16,
130
+ VLLM_DISPATCH_FLOATING_TYPES(
131
key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
132
vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
133
key_cache_ptrs_tensor.data_ptr<int64_t>(),
@@ -202,9 +202,7 @@ void reshape_and_cache(
202
dim3 grid(num_tokens);
203
dim3 block(std::min(num_heads * head_size, 512));
204
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
205
- AT_DISPATCH_FLOATING_TYPES_AND2(
206
- at::ScalarType::Half,
207
- at::ScalarType::BFloat16,
205
+ VLLM_DISPATCH_FLOATING_TYPES(
206
key.scalar_type(),
207
"reshape_and_cache_kernel",
208
[&] {
@@ -364,9 +362,7 @@ void gather_cached_kv(
362
dim3 grid(num_tokens);
363
dim3 block(std::min(num_heads * head_size, 512));
364
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
367
- AT_DISPATCH_FLOATING_TYPES_AND2(
368
- at::ScalarType::Half,
369
- at::ScalarType::BFloat16,
365
+ VLLM_DISPATCH_FLOATING_TYPES(
366
key.scalar_type(),
367
"gather_cached_kv_kernel_optimized",
368
[&] {
csrc/dispatch_utils.h
new
+14
@@ -0,0 +1,14 @@
1
+/*
2
+ * Adapted from
3
+ * https://github.com/pytorch/pytorch/blob/v2.0.1/aten/src/ATen/Dispatch.h
4
+ */
5
+#include <torch/extension.h>
6
+
7
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
8
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
9
+ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
10
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
11
+
12
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
13
+ AT_DISPATCH_SWITCH( \
14
+ TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
csrc/layernorm_kernels.cu
+2
-3
@@ -1,6 +1,7 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
+#include "dispatch_utils.h"
5
#include "reduction_utils.cuh"
6
7
namespace vllm {
@@ -46,9 +47,7 @@ void rms_norm(
47
dim3 grid(num_tokens);
48
dim3 block(std::min(hidden_size, 1024));
49
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
49
- AT_DISPATCH_FLOATING_TYPES_AND2(
50
- at::ScalarType::Half,
51
- at::ScalarType::BFloat16,
50
+ VLLM_DISPATCH_FLOATING_TYPES(
51
input.scalar_type(),
52
"rms_norm_kernel",
53
[&] {
csrc/pos_encoding_kernels.cu
+3
-3
@@ -1,6 +1,8 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
+#include "dispatch_utils.h"
5
+
6
namespace vllm {
7
8
template<typename scalar_t>
@@ -83,9 +85,7 @@ void rotary_embedding_neox(
85
dim3 grid(num_tokens);
86
dim3 block(std::min(num_heads * rot_dim / 2, 512));
87
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
86
- AT_DISPATCH_FLOATING_TYPES_AND2(
87
- at::ScalarType::Half,
88
- at::ScalarType::BFloat16,
88
+ VLLM_DISPATCH_FLOATING_TYPES(
89
query.scalar_type(),
90
"rotary_embedding_neox",
91
[&] {