@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8ce9c50d40
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Avoid compiling kernels for double data type (#933)

Woosuk Kwon committed Sep 2, 2023 at 14:59 UTC 8ce9c50d4034de3c557b520935fac1d6dac585a0
5 files changed +28 -21
csrc/activation_kernels.cu
+4 -6
@@ -1,6 +1,8 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 +#include "dispatch_utils.h"
5 +
6 namespace vllm {
7
8 template<typename T>
@@ -34,9 +36,7 @@ void silu_and_mul(
36 dim3 grid(num_tokens);
37 dim3 block(std::min(d, 1024));
38 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
37 - AT_DISPATCH_FLOATING_TYPES_AND2(
38 - at::ScalarType::Half,
39 - at::ScalarType::BFloat16,
39 + VLLM_DISPATCH_FLOATING_TYPES(
40 input.scalar_type(),
41 "silu_and_mul_kernel",
42 [&] {
@@ -71,9 +71,7 @@ __global__ void activation_kernel(
71 dim3 grid(num_tokens); \
72 dim3 block(std::min(d, 1024)); \
73 const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
74 - AT_DISPATCH_FLOATING_TYPES_AND2( \
75 - at::ScalarType::Half, \
76 - at::ScalarType::BFloat16, \
74 + VLLM_DISPATCH_FLOATING_TYPES( \
75 input.scalar_type(), \
76 "activation_kernel", \
77 [&] { \
csrc/cache_kernels.cu
+5 -9
@@ -1,6 +1,8 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 +#include "dispatch_utils.h"
5 +
6 #include <algorithm>
7 #include <cassert>
8 #include <map>
@@ -125,9 +127,7 @@ void copy_blocks(
127 dim3 grid(num_layers, num_pairs);
128 dim3 block(std::min(1024, numel_per_block));
129 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
128 - AT_DISPATCH_FLOATING_TYPES_AND2(
129 - at::ScalarType::Half,
130 - at::ScalarType::BFloat16,
130 + VLLM_DISPATCH_FLOATING_TYPES(
131 key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
132 vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
133 key_cache_ptrs_tensor.data_ptr<int64_t>(),
@@ -202,9 +202,7 @@ void reshape_and_cache(
202 dim3 grid(num_tokens);
203 dim3 block(std::min(num_heads * head_size, 512));
204 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
205 - AT_DISPATCH_FLOATING_TYPES_AND2(
206 - at::ScalarType::Half,
207 - at::ScalarType::BFloat16,
205 + VLLM_DISPATCH_FLOATING_TYPES(
206 key.scalar_type(),
207 "reshape_and_cache_kernel",
208 [&] {
@@ -364,9 +362,7 @@ void gather_cached_kv(
362 dim3 grid(num_tokens);
363 dim3 block(std::min(num_heads * head_size, 512));
364 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
367 - AT_DISPATCH_FLOATING_TYPES_AND2(
368 - at::ScalarType::Half,
369 - at::ScalarType::BFloat16,
365 + VLLM_DISPATCH_FLOATING_TYPES(
366 key.scalar_type(),
367 "gather_cached_kv_kernel_optimized",
368 [&] {
csrc/dispatch_utils.h new
+14
@@ -0,0 +1,14 @@
1 +/*
2 + * Adapted from
3 + * https://github.com/pytorch/pytorch/blob/v2.0.1/aten/src/ATen/Dispatch.h
4 + */
5 +#include <torch/extension.h>
6 +
7 +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
8 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
9 + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
10 + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
11 +
12 +#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
13 + AT_DISPATCH_SWITCH( \
14 + TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
csrc/layernorm_kernels.cu
+2 -3
@@ -1,6 +1,7 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 +#include "dispatch_utils.h"
5 #include "reduction_utils.cuh"
6
7 namespace vllm {
@@ -46,9 +47,7 @@ void rms_norm(
47 dim3 grid(num_tokens);
48 dim3 block(std::min(hidden_size, 1024));
49 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
49 - AT_DISPATCH_FLOATING_TYPES_AND2(
50 - at::ScalarType::Half,
51 - at::ScalarType::BFloat16,
50 + VLLM_DISPATCH_FLOATING_TYPES(
51 input.scalar_type(),
52 "rms_norm_kernel",
53 [&] {
csrc/pos_encoding_kernels.cu
+3 -3
@@ -1,6 +1,8 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 +#include "dispatch_utils.h"
5 +
6 namespace vllm {
7
8 template<typename scalar_t>
@@ -83,9 +85,7 @@ void rotary_embedding_neox(
85 dim3 grid(num_tokens);
86 dim3 block(std::min(num_heads * rot_dim / 2, 512));
87 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
86 - AT_DISPATCH_FLOATING_TYPES_AND2(
87 - at::ScalarType::Half,
88 - at::ScalarType::BFloat16,
88 + VLLM_DISPATCH_FLOATING_TYPES(
89 query.scalar_type(),
90 "rotary_embedding_neox",
91 [&] {