A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Kernel] Support MulAndSilu (#11624)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jan 15, 2025 at 10:29 UTC
42f5e7c52a5852e20937001332572c8cb8115af0
7 files changed
+83
-36
csrc/activation_kernels.cu
+25
-7
@@ -9,8 +9,16 @@
9
10
namespace vllm {
11
12
+template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
13
+ bool act_first>
14
+__device__ __forceinline__ scalar_t compute(const scalar_t& x,
15
+ const scalar_t& y) {
16
+ return act_first ? ACT_FN(x) * y : x * ACT_FN(y);
17
+}
18
// Activation and gating kernel template.
13
-template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
19
+
20
+template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
21
+ bool act_first>
22
__global__ void act_and_mul_kernel(
23
scalar_t* __restrict__ out, // [..., d]
24
const scalar_t* __restrict__ input, // [..., 2, d]
@@ -19,7 +27,7 @@ __global__ void act_and_mul_kernel(
27
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
28
const scalar_t x = VLLM_LDG(&input[token_idx * 2 * d + idx]);
29
const scalar_t y = VLLM_LDG(&input[token_idx * 2 * d + d + idx]);
22
- out[token_idx * d + idx] = ACT_FN(x) * y;
30
+ out[token_idx * d + idx] = compute<scalar_t, ACT_FN, act_first>(x, y);
31
}
32
}
33
@@ -55,7 +63,9 @@ __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
63
} // namespace vllm
64
65
// Launch activation and gating kernel.
58
-#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
66
+// Use ACT_FIRST (bool) indicating whether to apply the activation function
67
+// first.
68
+#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL, ACT_FIRST) \
69
int d = input.size(-1) / 2; \
70
int64_t num_tokens = input.numel() / input.size(-1); \
71
dim3 grid(num_tokens); \
@@ -64,7 +74,7 @@ __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
74
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
75
VLLM_DISPATCH_FLOATING_TYPES( \
76
input.scalar_type(), "act_and_mul_kernel", [&] { \
67
- vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>> \
77
+ vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>, ACT_FIRST> \
78
<<<grid, block, 0, stream>>>(out.data_ptr<scalar_t>(), \
79
input.data_ptr<scalar_t>(), d); \
80
});
@@ -72,19 +82,27 @@ __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
82
void silu_and_mul(torch::Tensor& out, // [..., d]
83
torch::Tensor& input) // [..., 2 * d]
84
{
75
- LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel);
85
+ LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, true);
86
+}
87
+
88
+void mul_and_silu(torch::Tensor& out, // [..., d]
89
+ torch::Tensor& input) // [..., 2 * d]
90
+{
91
+ // The difference between mul_and_silu and silu_and_mul is that mul_and_silu
92
+ // applies the silu to the latter half of the input.
93
+ LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, false);
94
}
95
96
void gelu_and_mul(torch::Tensor& out, // [..., d]
97
torch::Tensor& input) // [..., 2 * d]
98
{
81
- LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel);
99
+ LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel, true);
100
}
101
102
void gelu_tanh_and_mul(torch::Tensor& out, // [..., d]
103
torch::Tensor& input) // [..., 2 * d]
104
{
87
- LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel);
105
+ LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel, true);
106
}
107
108
namespace vllm {
csrc/ops.h
+2
@@ -86,6 +86,8 @@ void batched_rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
86
87
void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
88
89
+void mul_and_silu(torch::Tensor& out, torch::Tensor& input);
90
+
91
void gelu_and_mul(torch::Tensor& out, torch::Tensor& input);
92
93
void gelu_tanh_and_mul(torch::Tensor& out, torch::Tensor& input);
csrc/torch_bindings.cpp
+3
@@ -55,6 +55,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
55
ops.def("silu_and_mul(Tensor! out, Tensor input) -> ()");
56
ops.impl("silu_and_mul", torch::kCUDA, &silu_and_mul);
57
58
+ ops.def("mul_and_silu(Tensor! out, Tensor input) -> ()");
59
+ ops.impl("mul_and_silu", torch::kCUDA, &mul_and_silu);
60
+
61
// Activation function used in GeGLU with `none` approximation.
62
ops.def("gelu_and_mul(Tensor! out, Tensor input) -> ()");
63
ops.impl("gelu_and_mul", torch::kCUDA, &gelu_and_mul);
tests/kernels/test_activation.py
+13
-7
@@ -6,8 +6,9 @@ import torch
6
7
from tests.kernels.utils import opcheck
8
from vllm.model_executor.layers.activation import (FastGELU, FatreluAndMul,
9
- GeluAndMul, NewGELU,
10
- QuickGELU, SiluAndMul)
9
+ GeluAndMul, MulAndSilu,
10
+ NewGELU, QuickGELU,
11
+ SiluAndMul)
12
from vllm.platforms import current_platform
13
14
from .allclose_default import get_default_atol, get_default_rtol
@@ -21,8 +22,9 @@ CUDA_DEVICES = [
22
]
23
24
24
-@pytest.mark.parametrize("activation",
25
- ["silu", "gelu", "gelu_tanh", "fatrelu"])
25
+@pytest.mark.parametrize(
26
+ "activation",
27
+ ["silu_and_mul", "mul_and_silu", "gelu", "gelu_tanh", "fatrelu"])
28
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
29
@pytest.mark.parametrize("d", D)
30
@pytest.mark.parametrize("dtype", DTYPES)
@@ -40,9 +42,12 @@ def test_act_and_mul(
42
current_platform.seed_everything(seed)
43
torch.set_default_device(device)
44
x = torch.randn(num_tokens, 2 * d, dtype=dtype)
43
- if activation == "silu":
45
+ if activation == "silu_and_mul":
46
layer = SiluAndMul()
47
fn = torch.ops._C.silu_and_mul
48
+ if activation == "mul_and_silu":
49
+ layer = MulAndSilu()
50
+ fn = torch.ops._C.mul_and_silu
51
elif activation == "gelu":
52
layer = GeluAndMul(approximate="none")
53
fn = torch.ops._C.gelu_and_mul
@@ -55,8 +60,9 @@ def test_act_and_mul(
60
fn = torch.ops._C.fatrelu_and_mul
61
out = layer(x)
62
ref_out = layer.forward_native(x)
58
- # The SiLU, GELU and FatReLU implementations are equivalent to the native
59
- # PyTorch implementations, so we can do exact comparison.
63
+ # The SiluAndMul, MulAndSilu, GELU and FatReLU implementations are
64
+ # equivalent to the native PyTorch implementations, so we can do exact
65
+ # comparison.
66
torch.testing.assert_close(out, ref_out, atol=0.0, rtol=0.0)
67
68
d = x.shape[-1] // 2
vllm/model_executor/layers/activation.py
+35
@@ -87,6 +87,41 @@ class SiluAndMul(CustomOp):
87
return out
88
89
90
+@CustomOp.register("mul_and_silu")
91
+class MulAndSilu(CustomOp):
92
+ """An activation function for SwiGLU.
93
+
94
+ The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2.
95
+
96
+ Shapes:
97
+ x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
98
+ return: (num_tokens, d) or (batch_size, seq_len, d)
99
+ """
100
+
101
+ def __init__(self):
102
+ super().__init__()
103
+ if current_platform.is_cuda_alike() or current_platform.is_cpu():
104
+ self.op = torch.ops._C.mul_and_silu
105
+ elif current_platform.is_xpu():
106
+ from vllm._ipex_ops import ipex_ops
107
+ self.op = ipex_ops.silu_and_mul
108
+
109
+ def forward_native(self, x: torch.Tensor) -> torch.Tensor:
110
+ """PyTorch-native implementation equivalent to forward()."""
111
+ d = x.shape[-1] // 2
112
+ return x[..., :d] * F.silu(x[..., d:])
113
+
114
+ def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
115
+ d = x.shape[-1] // 2
116
+ output_shape = (x.shape[:-1] + (d, ))
117
+ out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
118
+ self.op(out, x)
119
+ return out
120
+
121
+ # TODO implement forward_xpu for MulAndSilu
122
+ # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
123
+
124
+
125
@CustomOp.register("gelu_and_mul")
126
class GeluAndMul(CustomOp):
127
"""An activation function for GeGLU.
vllm/model_executor/models/molmo.py
+3
-11
@@ -23,7 +23,8 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
23
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
24
InputContext, token_inputs)
25
from vllm.model_executor import SamplingMetadata
26
-from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
26
+from vllm.model_executor.layers.activation import (MulAndSilu, QuickGELU,
27
+ SiluAndMul)
28
from vllm.model_executor.layers.layernorm import RMSNorm
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
MergedColumnParallelLinear,
@@ -462,15 +463,6 @@ class MolmoAttention(nn.Module):
463
return output
464
465
465
-class SwiGLU(nn.Module):
466
-
467
- def forward(self, x: torch.Tensor) -> torch.Tensor:
468
- x, gate = x.chunk(2, dim=-1)
469
- # Note that the order is reversed compared to
470
- # SiluAndMul.
471
- return x * F.silu(gate)
472
-
473
-
466
class LanuageModelMLP(nn.Module):
467
"""Molmo's LLM mlp."""
468
@@ -489,7 +481,7 @@ class LanuageModelMLP(nn.Module):
481
quant_config=quant_config,
482
)
483
# Activation function.
492
- self.act_fn = SwiGLU()
484
+ self.act_fn = MulAndSilu()
485
# Feed-forward output projection.
486
self.down_proj = RowParallelLinear(
487
self.intermediate_size,
vllm/model_executor/models/ultravox.py
+2
-11
@@ -16,7 +16,7 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
16
from vllm import envs
17
from vllm.attention import AttentionMetadata
18
from vllm.config import VllmConfig
19
-from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
19
+from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
20
from vllm.model_executor.layers.layernorm import RMSNorm
21
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22
from vllm.model_executor.model_loader.loader import DefaultModelLoader
@@ -248,15 +248,6 @@ class StackAudioFrames(nn.Module):
248
return audio_embeds
249
250
251
-class FlippedSiluAndMul(SiluAndMul):
252
- """Ultravox is trained with SwiGLU with flipped halves."""
253
-
254
- def forward(self, x: torch.Tensor):
255
- a, b = x.chunk(2, dim=-1)
256
- flipped = torch.cat((b, a), dim=-1)
257
- return super().forward(flipped)
258
-
259
-
251
class UltravoxProjector(nn.Module):
252
253
def __init__(self, config: UltravoxConfig):
@@ -269,7 +260,7 @@ class UltravoxProjector(nn.Module):
260
dim = self.hidden_dim
261
262
if config.projector_act == "swiglu":
272
- self.act = FlippedSiluAndMul()
263
+ self.act = MulAndSilu()
264
dim = dim // 2
265
else:
266
self.act = get_act_fn(config.projector_act)